2024-11-09 10:15:22,530 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 10:15:22,541 main DEBUG Took 0.009451 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 10:15:22,542 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 10:15:22,542 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 10:15:22,543 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 10:15:22,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,551 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 10:15:22,563 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,566 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,567 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,568 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,569 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,569 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,570 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,571 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,571 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,571 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,572 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,572 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,572 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,573 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,573 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 10:15:22,573 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,574 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 10:15:22,575 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 10:15:22,576 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 10:15:22,578 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 10:15:22,579 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 10:15:22,580 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 10:15:22,580 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 10:15:22,588 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 10:15:22,590 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 10:15:22,592 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 10:15:22,592 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 10:15:22,592 main DEBUG createAppenders(={Console}) 2024-11-09 10:15:22,593 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-09 10:15:22,593 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 10:15:22,594 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-09 10:15:22,594 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 10:15:22,594 main DEBUG OutputStream closed 2024-11-09 10:15:22,595 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 10:15:22,595 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 10:15:22,595 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-09 10:15:22,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 10:15:22,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 10:15:22,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 10:15:22,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 10:15:22,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 10:15:22,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 10:15:22,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 10:15:22,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 10:15:22,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 10:15:22,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 10:15:22,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 10:15:22,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 10:15:22,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 10:15:22,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 10:15:22,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 10:15:22,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 10:15:22,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 10:15:22,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 10:15:22,687 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 10:15:22,687 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-09 10:15:22,688 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 10:15:22,688 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-09T10:15:22,947 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2 2024-11-09 10:15:22,950 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 10:15:22,950 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T10:15:22,958 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-09T10:15:22,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T10:15:22,998 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4, deleteOnExit=true 2024-11-09T10:15:22,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T10:15:23,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/test.cache.data in system properties and HBase conf 2024-11-09T10:15:23,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T10:15:23,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/hadoop.log.dir in system properties and HBase conf 2024-11-09T10:15:23,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T10:15:23,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T10:15:23,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T10:15:23,093 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T10:15:23,175 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T10:15:23,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T10:15:23,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T10:15:23,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T10:15:23,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T10:15:23,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T10:15:23,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T10:15:23,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T10:15:23,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T10:15:23,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T10:15:23,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/nfs.dump.dir in system properties and HBase conf 2024-11-09T10:15:23,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/java.io.tmpdir in system properties and HBase conf 2024-11-09T10:15:23,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T10:15:23,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T10:15:23,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T10:15:24,321 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T10:15:24,394 INFO [Time-limited test {}] log.Log(170): Logging initialized @2540ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T10:15:24,466 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T10:15:24,522 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T10:15:24,542 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T10:15:24,542 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T10:15:24,543 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T10:15:24,555 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T10:15:24,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/hadoop.log.dir/,AVAILABLE} 2024-11-09T10:15:24,559 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T10:15:24,726 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/java.io.tmpdir/jetty-localhost-37729-hadoop-hdfs-3_4_1-tests_jar-_-any-17532114675448024625/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T10:15:24,737 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:37729} 2024-11-09T10:15:24,737 INFO [Time-limited test {}] server.Server(415): Started @2884ms 2024-11-09T10:15:25,297 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T10:15:25,304 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T10:15:25,305 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T10:15:25,305 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T10:15:25,305 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T10:15:25,306 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/hadoop.log.dir/,AVAILABLE} 2024-11-09T10:15:25,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T10:15:25,407 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/java.io.tmpdir/jetty-localhost-44733-hadoop-hdfs-3_4_1-tests_jar-_-any-13764622969060765242/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T10:15:25,407 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:44733} 2024-11-09T10:15:25,408 INFO [Time-limited test {}] server.Server(415): Started @3554ms 2024-11-09T10:15:25,455 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T10:15:26,792 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4/data/data2/current/BP-2048340984-172.17.0.3-1731147323772/current, will proceed with Du for space computation calculation, 2024-11-09T10:15:26,792 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4/data/data1/current/BP-2048340984-172.17.0.3-1731147323772/current, will proceed with Du for space computation calculation, 2024-11-09T10:15:26,819 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T10:15:26,865 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x35ee918b675aa609 with lease ID 0x70178ccd39ffa2b8: Processing first storage report for DS-194c2f31-fcc8-4d74-88cb-28d08ed597ab from datanode DatanodeRegistration(127.0.0.1:37425, datanodeUuid=a9670b94-06a3-4eea-93ae-1a8a605d573e, infoPort=36409, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1371902043;c=1731147323772) 2024-11-09T10:15:26,867 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x35ee918b675aa609 with lease ID 0x70178ccd39ffa2b8: from storage DS-194c2f31-fcc8-4d74-88cb-28d08ed597ab node DatanodeRegistration(127.0.0.1:37425, datanodeUuid=a9670b94-06a3-4eea-93ae-1a8a605d573e, infoPort=36409, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1371902043;c=1731147323772), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T10:15:26,867 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x35ee918b675aa609 with lease ID 0x70178ccd39ffa2b8: Processing first storage report for DS-7d639b2d-c048-44ac-8cef-ad354674fc91 from datanode DatanodeRegistration(127.0.0.1:37425, datanodeUuid=a9670b94-06a3-4eea-93ae-1a8a605d573e, infoPort=36409, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1371902043;c=1731147323772) 2024-11-09T10:15:26,867 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x35ee918b675aa609 with lease ID 0x70178ccd39ffa2b8: from storage DS-7d639b2d-c048-44ac-8cef-ad354674fc91 node DatanodeRegistration(127.0.0.1:37425, datanodeUuid=a9670b94-06a3-4eea-93ae-1a8a605d573e, infoPort=36409, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1371902043;c=1731147323772), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T10:15:26,958 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2 2024-11-09T10:15:27,025 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4/zookeeper_0, clientPort=59425, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T10:15:27,035 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59425 2024-11-09T10:15:27,044 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T10:15:27,047 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T10:15:27,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741825_1001 (size=7) 2024-11-09T10:15:27,670 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 with version=8 2024-11-09T10:15:27,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/hbase-staging 2024-11-09T10:15:27,755 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T10:15:28,057 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e067b2c2c7b3:0 server-side Connection retries=45 2024-11-09T10:15:28,067 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T10:15:28,067 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T10:15:28,072 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T10:15:28,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T10:15:28,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T10:15:28,245 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T10:15:28,317 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T10:15:28,327 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T10:15:28,331 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T10:15:28,358 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8242 (auto-detected) 2024-11-09T10:15:28,359 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-09T10:15:28,379 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42133 2024-11-09T10:15:28,404 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42133 connecting to ZooKeeper ensemble=127.0.0.1:59425 2024-11-09T10:15:28,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:421330x0, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T10:15:28,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42133-0x1011f17e1cc0000 connected 2024-11-09T10:15:28,631 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T10:15:28,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T10:15:28,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T10:15:28,645 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51, hbase.cluster.distributed=false 2024-11-09T10:15:28,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T10:15:28,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42133 2024-11-09T10:15:28,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42133 2024-11-09T10:15:28,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42133 2024-11-09T10:15:28,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42133 2024-11-09T10:15:28,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42133 2024-11-09T10:15:28,790 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e067b2c2c7b3:0 server-side Connection retries=45 2024-11-09T10:15:28,791 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T10:15:28,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T10:15:28,792 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T10:15:28,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T10:15:28,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T10:15:28,795 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T10:15:28,797 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T10:15:28,798 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38427 2024-11-09T10:15:28,800 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38427 connecting to ZooKeeper ensemble=127.0.0.1:59425 2024-11-09T10:15:28,801 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T10:15:28,804 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T10:15:28,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384270x0, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T10:15:28,913 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38427-0x1011f17e1cc0001 connected 2024-11-09T10:15:28,914 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T10:15:28,920 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T10:15:28,932 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T10:15:28,940 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T10:15:28,950 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T10:15:28,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38427 2024-11-09T10:15:28,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38427 2024-11-09T10:15:28,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38427 2024-11-09T10:15:28,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38427 2024-11-09T10:15:28,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38427 2024-11-09T10:15:28,993 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e067b2c2c7b3:42133 2024-11-09T10:15:28,994 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:29,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T10:15:29,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T10:15:29,013 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T10:15:29,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:29,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:29,055 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T10:15:29,058 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e067b2c2c7b3,42133,1731147327850 from backup master directory 2024-11-09T10:15:29,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:29,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T10:15:29,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T10:15:29,076 WARN [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T10:15:29,077 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:29,080 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T10:15:29,082 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T10:15:29,164 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/hbase.id] with ID: e4d84759-b9d8-4b81-ab08-fb22b416de6b 2024-11-09T10:15:29,165 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/.tmp/hbase.id 2024-11-09T10:15:29,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741826_1002 (size=42) 2024-11-09T10:15:29,181 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/.tmp/hbase.id]:[hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/hbase.id] 2024-11-09T10:15:29,252 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T10:15:29,258 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T10:15:29,284 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-09T10:15:29,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:29,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:29,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741827_1003 (size=196) 2024-11-09T10:15:29,343 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:15:29,347 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T10:15:29,370 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:29,377 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T10:15:29,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741828_1004 (size=1189) 2024-11-09T10:15:29,843 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store 2024-11-09T10:15:29,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741829_1005 (size=34) 2024-11-09T10:15:30,271 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T10:15:30,275 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:15:30,276 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T10:15:30,276 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:15:30,277 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:15:30,278 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T10:15:30,278 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:15:30,279 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:15:30,280 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731147330276Disabling compacts and flushes for region at 1731147330276Disabling writes for close at 1731147330278 (+2 ms)Writing region close event to WAL at 1731147330279 (+1 ms)Closed at 1731147330279 2024-11-09T10:15:30,282 WARN [master/e067b2c2c7b3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/.initializing 2024-11-09T10:15:30,282 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/WALs/e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:30,291 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T10:15:30,304 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e067b2c2c7b3%2C42133%2C1731147327850, suffix=, logDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/WALs/e067b2c2c7b3,42133,1731147327850, archiveDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/oldWALs, maxLogs=10 2024-11-09T10:15:30,325 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/WALs/e067b2c2c7b3,42133,1731147327850/e067b2c2c7b3%2C42133%2C1731147327850.1731147330309, exclude list is [], retry=0 2024-11-09T10:15:30,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37425,DS-194c2f31-fcc8-4d74-88cb-28d08ed597ab,DISK] 2024-11-09T10:15:30,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T10:15:30,374 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/WALs/e067b2c2c7b3,42133,1731147327850/e067b2c2c7b3%2C42133%2C1731147327850.1731147330309 2024-11-09T10:15:30,375 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36409:36409)] 2024-11-09T10:15:30,376 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:15:30,376 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:15:30,379 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,380 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,414 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T10:15:30,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:30,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:30,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T10:15:30,447 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:30,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:15:30,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T10:15:30,451 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:30,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:15:30,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T10:15:30,455 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:30,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:15:30,456 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,460 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,461 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,466 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,467 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,470 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T10:15:30,474 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T10:15:30,479 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:15:30,480 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64725775, jitterRate=-0.03551079332828522}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T10:15:30,487 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731147330391Initializing all the Stores at 1731147330393 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731147330393Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147330394 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147330394Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147330394Cleaning up temporary data from old regions at 1731147330467 (+73 ms)Region opened successfully at 1731147330487 (+20 ms) 2024-11-09T10:15:30,489 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T10:15:30,520 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8efb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e067b2c2c7b3/172.17.0.3:0 2024-11-09T10:15:30,547 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T10:15:30,557 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T10:15:30,557 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T10:15:30,560 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T10:15:30,562 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T10:15:30,566 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T10:15:30,567 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T10:15:30,589 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T10:15:30,597 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T10:15:30,651 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T10:15:30,655 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T10:15:30,657 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T10:15:30,666 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T10:15:30,670 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T10:15:30,675 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T10:15:30,687 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T10:15:30,689 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T10:15:30,698 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T10:15:30,720 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T10:15:30,729 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T10:15:30,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T10:15:30,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T10:15:30,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:30,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:30,744 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e067b2c2c7b3,42133,1731147327850, sessionid=0x1011f17e1cc0000, setting cluster-up flag (Was=false) 2024-11-09T10:15:30,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:30,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:30,809 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T10:15:30,814 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:30,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:30,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:30,872 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T10:15:30,874 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:30,882 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T10:15:30,949 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T10:15:30,959 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T10:15:30,965 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T10:15:30,970 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e067b2c2c7b3,42133,1731147327850 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T10:15:30,973 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(746): ClusterId : e4d84759-b9d8-4b81-ab08-fb22b416de6b 2024-11-09T10:15:30,975 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T10:15:30,978 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e067b2c2c7b3:0, corePoolSize=5, maxPoolSize=5 2024-11-09T10:15:30,978 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e067b2c2c7b3:0, corePoolSize=5, maxPoolSize=5 2024-11-09T10:15:30,978 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e067b2c2c7b3:0, corePoolSize=5, maxPoolSize=5 2024-11-09T10:15:30,978 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e067b2c2c7b3:0, corePoolSize=5, maxPoolSize=5 2024-11-09T10:15:30,979 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e067b2c2c7b3:0, corePoolSize=10, maxPoolSize=10 2024-11-09T10:15:30,979 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:30,979 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e067b2c2c7b3:0, corePoolSize=2, maxPoolSize=2 2024-11-09T10:15:30,979 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:30,984 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731147360984 2024-11-09T10:15:30,984 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T10:15:30,985 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T10:15:30,985 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T10:15:30,986 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T10:15:30,989 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T10:15:30,989 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T10:15:30,989 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T10:15:30,989 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T10:15:30,990 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T10:15:30,990 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T10:15:30,992 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:30,991 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:30,992 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T10:15:30,994 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T10:15:30,995 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T10:15:30,995 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T10:15:30,997 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T10:15:30,998 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T10:15:30,999 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T10:15:31,000 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e067b2c2c7b3:0:becomeActiveMaster-HFileCleaner.large.0-1731147330999,5,FailOnTimeoutGroup] 2024-11-09T10:15:31,000 DEBUG [RS:0;e067b2c2c7b3:38427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a97373b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e067b2c2c7b3/172.17.0.3:0 2024-11-09T10:15:31,004 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e067b2c2c7b3:0:becomeActiveMaster-HFileCleaner.small.0-1731147331000,5,FailOnTimeoutGroup] 2024-11-09T10:15:31,004 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,004 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T10:15:31,006 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,006 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741831_1007 (size=1321) 2024-11-09T10:15:31,018 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e067b2c2c7b3:38427 2024-11-09T10:15:31,020 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T10:15:31,021 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T10:15:31,021 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T10:15:31,023 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(2659): reportForDuty to master=e067b2c2c7b3,42133,1731147327850 with port=38427, startcode=1731147328756 2024-11-09T10:15:31,032 DEBUG [RS:0;e067b2c2c7b3:38427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T10:15:31,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55747, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T10:15:31,066 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42133 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:31,068 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42133 {}] master.ServerManager(517): Registering regionserver=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:31,080 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:15:31,080 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46023 2024-11-09T10:15:31,080 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T10:15:31,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T10:15:31,093 DEBUG [RS:0;e067b2c2c7b3:38427 {}] zookeeper.ZKUtil(111): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:31,094 WARN [RS:0;e067b2c2c7b3:38427 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T10:15:31,094 INFO [RS:0;e067b2c2c7b3:38427 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T10:15:31,094 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/WALs/e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:31,096 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e067b2c2c7b3,38427,1731147328756] 2024-11-09T10:15:31,116 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T10:15:31,131 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T10:15:31,136 INFO [RS:0;e067b2c2c7b3:38427 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T10:15:31,136 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,137 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T10:15:31,142 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T10:15:31,143 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,143 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,143 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,143 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,143 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,144 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,144 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e067b2c2c7b3:0, corePoolSize=2, maxPoolSize=2 2024-11-09T10:15:31,144 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,144 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,144 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,144 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,144 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,145 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e067b2c2c7b3:0, corePoolSize=1, maxPoolSize=1 2024-11-09T10:15:31,145 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e067b2c2c7b3:0, corePoolSize=3, maxPoolSize=3 2024-11-09T10:15:31,145 DEBUG [RS:0;e067b2c2c7b3:38427 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0, corePoolSize=3, maxPoolSize=3 2024-11-09T10:15:31,146 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,146 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,147 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,147 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,147 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,147 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,38427,1731147328756-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T10:15:31,164 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T10:15:31,165 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,38427,1731147328756-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,166 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,166 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.Replication(171): e067b2c2c7b3,38427,1731147328756 started 2024-11-09T10:15:31,183 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:31,184 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1482): Serving as e067b2c2c7b3,38427,1731147328756, RpcServer on e067b2c2c7b3/172.17.0.3:38427, sessionid=0x1011f17e1cc0001 2024-11-09T10:15:31,184 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T10:15:31,184 DEBUG [RS:0;e067b2c2c7b3:38427 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:31,185 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e067b2c2c7b3,38427,1731147328756' 2024-11-09T10:15:31,185 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T10:15:31,186 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T10:15:31,186 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T10:15:31,186 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T10:15:31,187 DEBUG [RS:0;e067b2c2c7b3:38427 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:31,187 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e067b2c2c7b3,38427,1731147328756' 2024-11-09T10:15:31,187 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T10:15:31,187 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T10:15:31,188 DEBUG [RS:0;e067b2c2c7b3:38427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T10:15:31,188 INFO [RS:0;e067b2c2c7b3:38427 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T10:15:31,188 INFO [RS:0;e067b2c2c7b3:38427 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T10:15:31,295 INFO [RS:0;e067b2c2c7b3:38427 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T10:15:31,299 INFO [RS:0;e067b2c2c7b3:38427 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e067b2c2c7b3%2C38427%2C1731147328756, suffix=, logDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/WALs/e067b2c2c7b3,38427,1731147328756, archiveDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/oldWALs, maxLogs=32 2024-11-09T10:15:31,313 DEBUG [RS:0;e067b2c2c7b3:38427 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/WALs/e067b2c2c7b3,38427,1731147328756/e067b2c2c7b3%2C38427%2C1731147328756.1731147331301, exclude list is [], retry=0 2024-11-09T10:15:31,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37425,DS-194c2f31-fcc8-4d74-88cb-28d08ed597ab,DISK] 2024-11-09T10:15:31,321 INFO [RS:0;e067b2c2c7b3:38427 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/WALs/e067b2c2c7b3,38427,1731147328756/e067b2c2c7b3%2C38427%2C1731147328756.1731147331301 2024-11-09T10:15:31,321 DEBUG [RS:0;e067b2c2c7b3:38427 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36409:36409)] 2024-11-09T10:15:31,409 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T10:15:31,409 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:15:31,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741833_1009 (size=32) 2024-11-09T10:15:31,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:15:31,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T10:15:31,833 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T10:15:31,833 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:31,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:31,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T10:15:31,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T10:15:31,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:31,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:31,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T10:15:31,842 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T10:15:31,842 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:31,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:31,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T10:15:31,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T10:15:31,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:31,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:31,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T10:15:31,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740 2024-11-09T10:15:31,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740 2024-11-09T10:15:31,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T10:15:31,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T10:15:31,852 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:15:31,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T10:15:31,857 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:15:31,858 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60749664, jitterRate=-0.09475946426391602}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:15:31,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731147331823Initializing all the Stores at 1731147331826 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731147331827 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731147331827Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147331827Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731147331827Cleaning up temporary data from old regions at 1731147331851 (+24 ms)Region opened successfully at 1731147331861 (+10 ms) 2024-11-09T10:15:31,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T10:15:31,861 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T10:15:31,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T10:15:31,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T10:15:31,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T10:15:31,862 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T10:15:31,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731147331861Disabling compacts and flushes for region at 1731147331861Disabling writes for close at 1731147331861Writing region close event to WAL at 1731147331862 (+1 ms)Closed at 1731147331862 2024-11-09T10:15:31,866 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T10:15:31,866 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T10:15:31,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T10:15:31,878 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T10:15:31,881 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T10:15:32,038 DEBUG [e067b2c2c7b3:42133 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T10:15:32,049 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:32,055 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e067b2c2c7b3,38427,1731147328756, state=OPENING 2024-11-09T10:15:32,214 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T10:15:32,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:32,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:15:32,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T10:15:32,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T10:15:32,292 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T10:15:32,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:15:32,476 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T10:15:32,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53213, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T10:15:32,489 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T10:15:32,490 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T10:15:32,490 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T10:15:32,494 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e067b2c2c7b3%2C38427%2C1731147328756.meta, suffix=.meta, logDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/WALs/e067b2c2c7b3,38427,1731147328756, archiveDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/oldWALs, maxLogs=32 2024-11-09T10:15:32,509 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/WALs/e067b2c2c7b3,38427,1731147328756/e067b2c2c7b3%2C38427%2C1731147328756.meta.1731147332496.meta, exclude list is [], retry=0 2024-11-09T10:15:32,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37425,DS-194c2f31-fcc8-4d74-88cb-28d08ed597ab,DISK] 2024-11-09T10:15:32,516 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/WALs/e067b2c2c7b3,38427,1731147328756/e067b2c2c7b3%2C38427%2C1731147328756.meta.1731147332496.meta 2024-11-09T10:15:32,516 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36409:36409)] 2024-11-09T10:15:32,517 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:15:32,518 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T10:15:32,521 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T10:15:32,525 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T10:15:32,528 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T10:15:32,529 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:15:32,529 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T10:15:32,529 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T10:15:32,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T10:15:32,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T10:15:32,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:32,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:32,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T10:15:32,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T10:15:32,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:32,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:32,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T10:15:32,538 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T10:15:32,538 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:32,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:32,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T10:15:32,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T10:15:32,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:32,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T10:15:32,542 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T10:15:32,544 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740 2024-11-09T10:15:32,547 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740 2024-11-09T10:15:32,549 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T10:15:32,550 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T10:15:32,551 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:15:32,554 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T10:15:32,556 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66238263, jitterRate=-0.012972965836524963}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:15:32,556 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T10:15:32,558 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731147332530Writing region info on filesystem at 1731147332530Initializing all the Stores at 1731147332531 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731147332532 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731147332532Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147332532Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731147332532Cleaning up temporary data from old regions at 1731147332550 (+18 ms)Running coprocessor post-open hooks at 1731147332556 (+6 ms)Region opened successfully at 1731147332558 (+2 ms) 2024-11-09T10:15:32,564 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731147332465 2024-11-09T10:15:32,574 DEBUG [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T10:15:32,574 INFO [RS_OPEN_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T10:15:32,576 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:32,578 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e067b2c2c7b3,38427,1731147328756, state=OPEN 2024-11-09T10:15:32,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T10:15:32,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T10:15:32,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T10:15:32,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T10:15:32,773 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:32,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T10:15:32,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e067b2c2c7b3,38427,1731147328756 in 480 msec 2024-11-09T10:15:32,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T10:15:32,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 913 msec 2024-11-09T10:15:32,789 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T10:15:32,789 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T10:15:32,807 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:32,808 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:32,826 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:32,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56045, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:32,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.9340 sec 2024-11-09T10:15:32,846 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731147332846, completionTime=-1 2024-11-09T10:15:32,849 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T10:15:32,849 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T10:15:32,874 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-09T10:15:32,874 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731147392874 2024-11-09T10:15:32,874 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731147452874 2024-11-09T10:15:32,874 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-09T10:15:32,876 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,42133,1731147327850-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:32,876 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,42133,1731147327850-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:32,876 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,42133,1731147327850-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:32,878 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e067b2c2c7b3:42133, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:32,878 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:32,879 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:32,884 DEBUG [master/e067b2c2c7b3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T10:15:32,903 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.826sec 2024-11-09T10:15:32,904 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T10:15:32,906 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T10:15:32,907 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T10:15:32,907 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T10:15:32,907 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T10:15:32,908 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,42133,1731147327850-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T10:15:32,909 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,42133,1731147327850-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T10:15:32,916 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T10:15:32,917 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T10:15:32,918 INFO [master/e067b2c2c7b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e067b2c2c7b3,42133,1731147327850-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T10:15:32,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:32,988 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T10:15:32,988 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T10:15:32,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:32,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:33,001 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:33,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:33,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:33,016 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:33,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:33,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:33,058 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=1350 2024-11-09T10:15:33,066 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T10:15:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-11-09T10:15:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T10:15:33,092 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T10:15:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:15:33,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:15:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:33,110 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T10:15:33,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-09T10:15:33,112 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:33,115 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T10:15:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T10:15:33,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741835_1011 (size=1061) 2024-11-09T10:15:33,136 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:15:33,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741836_1012 (size=53) 2024-11-09T10:15:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T10:15:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T10:15:33,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:15:33,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cac2483be99c556ef585845f10246658, disabling compactions & flushes 2024-11-09T10:15:33,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. after waiting 0 ms 2024-11-09T10:15:33,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cac2483be99c556ef585845f10246658: Waiting for close lock at 1731147333549Disabling compacts and flushes for region at 1731147333549Disabling writes for close at 1731147333549Writing region close event to WAL at 1731147333549Closed at 1731147333549 2024-11-09T10:15:33,553 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T10:15:33,562 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731147333554"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731147333554"}]},"ts":"1731147333554"} 2024-11-09T10:15:33,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T10:15:33,568 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T10:15:33,570 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147333568"}]},"ts":"1731147333568"} 2024-11-09T10:15:33,575 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T10:15:33,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, ASSIGN}] 2024-11-09T10:15:33,579 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, ASSIGN 2024-11-09T10:15:33,581 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, ASSIGN; state=OFFLINE, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=false 2024-11-09T10:15:33,734 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cac2483be99c556ef585845f10246658, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:33,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, ASSIGN because future has completed 2024-11-09T10:15:33,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:15:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T10:15:33,909 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,909 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:15:33,910 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,910 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:15:33,910 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,910 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,913 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,917 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:15:33,918 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cac2483be99c556ef585845f10246658 columnFamilyName A 2024-11-09T10:15:33,918 DEBUG [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:33,920 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.HStore(327): Store=cac2483be99c556ef585845f10246658/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:15:33,920 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,922 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:15:33,922 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cac2483be99c556ef585845f10246658 columnFamilyName B 2024-11-09T10:15:33,923 DEBUG [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:33,923 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.HStore(327): Store=cac2483be99c556ef585845f10246658/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:15:33,924 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,926 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:15:33,926 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cac2483be99c556ef585845f10246658 columnFamilyName C 2024-11-09T10:15:33,926 DEBUG [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:33,927 INFO [StoreOpener-cac2483be99c556ef585845f10246658-1 {}] regionserver.HStore(327): Store=cac2483be99c556ef585845f10246658/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:15:33,928 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,928 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,929 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,930 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,932 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,932 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,933 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:15:33,935 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,938 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:15:33,939 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cac2483be99c556ef585845f10246658; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63654118, jitterRate=-0.051479727029800415}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:15:33,940 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cac2483be99c556ef585845f10246658 2024-11-09T10:15:33,940 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cac2483be99c556ef585845f10246658: Running coprocessor pre-open hook at 1731147333910Writing region info on filesystem at 1731147333911 (+1 ms)Initializing all the Stores at 1731147333912 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147333912Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147333913 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147333913Cleaning up temporary data from old regions at 1731147333932 (+19 ms)Running coprocessor post-open hooks at 1731147333940 (+8 ms)Region opened successfully at 1731147333940 2024-11-09T10:15:33,943 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., pid=6, masterSystemTime=1731147333900 2024-11-09T10:15:33,946 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,946 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:33,947 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cac2483be99c556ef585845f10246658, regionState=OPEN, openSeqNum=2, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:33,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:15:33,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T10:15:33,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 in 209 msec 2024-11-09T10:15:33,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T10:15:33,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, ASSIGN in 383 msec 2024-11-09T10:15:33,966 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T10:15:33,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147333966"}]},"ts":"1731147333966"} 2024-11-09T10:15:33,970 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T10:15:33,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T10:15:33,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 871 msec 2024-11-09T10:15:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T10:15:34,263 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:34,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,275 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1cdbfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,279 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@972c14f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,281 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,288 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T10:15:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e067b2c2c7b3,42133,1731147327850 2024-11-09T10:15:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1432456c 2024-11-09T10:15:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T10:15:34,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T10:15:34,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,299 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bb8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,305 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,315 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,319 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b3fb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,325 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,327 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,331 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,333 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5a398e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,338 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7489f107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,340 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333dd90d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,343 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d00fff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,346 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa1c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:15:34,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:15:34,349 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e082570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:34,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:15:34,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc1ac04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:15:34,358 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,358 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,359 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,360 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,360 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,360 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:34,361 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,363 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,366 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,367 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:15:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:15:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:15:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-09T10:15:34,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:15:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,375 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T10:15:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,379 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:34,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:15:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:34,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:15:34,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:34,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:34,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:34,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T10:15:34,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/afa85605a5d54f9da7e9a9dc2e52a4dd is 50, key is test_row_0/A:col10/1731147334432/Put/seqid=0 2024-11-09T10:15:34,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:34,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:34,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:34,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:34,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741837_1013 (size=12001) 2024-11-09T10:15:34,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/afa85605a5d54f9da7e9a9dc2e52a4dd 2024-11-09T10:15:34,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147334627, max timestamp=1731147334649Num uniques 859; 2024-11-09T10:15:34,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147334627, max timestamp=1731147334643Num uniques 859; 2024-11-09T10:15:34,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:15:34,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:15:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147394639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147334627, max timestamp=1731147334649Num uniques 859; 2024-11-09T10:15:34,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:15:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T10:15:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:34,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:34,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147394737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:34,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147394753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147394759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147394760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/e720c0b80eba45648bd2a3d801a314b2 is 50, key is test_row_0/B:col10/1731147334432/Put/seqid=0 2024-11-09T10:15:34,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741838_1014 (size=12001) 2024-11-09T10:15:34,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/e720c0b80eba45648bd2a3d801a314b2 2024-11-09T10:15:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147394820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2503f530f669445ab32be5438bb73adb is 50, key is test_row_0/C:col10/1731147334432/Put/seqid=0 2024-11-09T10:15:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147394870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147394880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147394881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147394882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:34,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741839_1015 (size=12001) 2024-11-09T10:15:34,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:34,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T10:15:35,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147395030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:35,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147395079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147395100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147395100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147395101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2503f530f669445ab32be5438bb73adb 2024-11-09T10:15:35,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/afa85605a5d54f9da7e9a9dc2e52a4dd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/afa85605a5d54f9da7e9a9dc2e52a4dd 2024-11-09T10:15:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147395340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/afa85605a5d54f9da7e9a9dc2e52a4dd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T10:15:35,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/e720c0b80eba45648bd2a3d801a314b2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e720c0b80eba45648bd2a3d801a314b2 2024-11-09T10:15:35,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e720c0b80eba45648bd2a3d801a314b2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T10:15:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2503f530f669445ab32be5438bb73adb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2503f530f669445ab32be5438bb73adb 2024-11-09T10:15:35,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:35,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:35,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147395390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2503f530f669445ab32be5438bb73adb, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T10:15:35,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cac2483be99c556ef585845f10246658 in 948ms, sequenceid=12, compaction requested=false 2024-11-09T10:15:35,405 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T10:15:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:35,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-09T10:15:35,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:35,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:35,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:35,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:35,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:35,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:35,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b54d28bc7e9a42248485df1265ee268a is 50, key is test_row_0/A:col10/1731147334726/Put/seqid=0 2024-11-09T10:15:35,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741840_1016 (size=14341) 2024-11-09T10:15:35,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b54d28bc7e9a42248485df1265ee268a 2024-11-09T10:15:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147395462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147395465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147395469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T10:15:35,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/7843c7eaed2f4cd293d22a998d5f0921 is 50, key is test_row_0/B:col10/1731147334726/Put/seqid=0 2024-11-09T10:15:35,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741841_1017 (size=12001) 2024-11-09T10:15:35,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/7843c7eaed2f4cd293d22a998d5f0921 2024-11-09T10:15:35,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:35,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:35,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:35,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2b324bc37ada4624b022140725ae7843 is 50, key is test_row_0/C:col10/1731147334726/Put/seqid=0 2024-11-09T10:15:35,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741842_1018 (size=12001) 2024-11-09T10:15:35,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2b324bc37ada4624b022140725ae7843 2024-11-09T10:15:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b54d28bc7e9a42248485df1265ee268a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b54d28bc7e9a42248485df1265ee268a 2024-11-09T10:15:35,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147395600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147395601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147395602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b54d28bc7e9a42248485df1265ee268a, entries=200, sequenceid=49, filesize=14.0 K 2024-11-09T10:15:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/7843c7eaed2f4cd293d22a998d5f0921 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7843c7eaed2f4cd293d22a998d5f0921 2024-11-09T10:15:35,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7843c7eaed2f4cd293d22a998d5f0921, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T10:15:35,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2b324bc37ada4624b022140725ae7843 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2b324bc37ada4624b022140725ae7843 2024-11-09T10:15:35,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2b324bc37ada4624b022140725ae7843, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T10:15:35,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cac2483be99c556ef585845f10246658 in 246ms, sequenceid=49, compaction requested=false 2024-11-09T10:15:35,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:35,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T10:15:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:35,706 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:35,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e6b9026ecbf042939f4e14b924ace161 is 50, key is test_row_0/A:col10/1731147335464/Put/seqid=0 2024-11-09T10:15:35,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741843_1019 (size=12001) 2024-11-09T10:15:35,740 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e6b9026ecbf042939f4e14b924ace161 2024-11-09T10:15:35,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/41cd2f397b914cdcbf07109dd6458299 is 50, key is test_row_0/B:col10/1731147335464/Put/seqid=0 2024-11-09T10:15:35,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741844_1020 (size=12001) 2024-11-09T10:15:35,802 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/41cd2f397b914cdcbf07109dd6458299 2024-11-09T10:15:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:35,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:35,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/138c260325e14d2e889c1fde2682e0a5 is 50, key is test_row_0/C:col10/1731147335464/Put/seqid=0 2024-11-09T10:15:35,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741845_1021 (size=12001) 2024-11-09T10:15:35,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147335874, max timestamp=1731147335896Num uniques 859; 2024-11-09T10:15:35,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:15:35,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147335874, max timestamp=1731147335902Num uniques 859; 2024-11-09T10:15:35,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:15:35,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731147335874, max timestamp=1731147335896Num uniques 859; 2024-11-09T10:15:35,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T10:15:35,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147395951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147395951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147395951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147395955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147395955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147396080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147396080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147396080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147396080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147396080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,260 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/138c260325e14d2e889c1fde2682e0a5 2024-11-09T10:15:36,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e6b9026ecbf042939f4e14b924ace161 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6b9026ecbf042939f4e14b924ace161 2024-11-09T10:15:36,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147396289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147396289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147396290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,299 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6b9026ecbf042939f4e14b924ace161, entries=150, sequenceid=59, filesize=11.7 K 2024-11-09T10:15:36,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147396290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147396290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/41cd2f397b914cdcbf07109dd6458299 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/41cd2f397b914cdcbf07109dd6458299 2024-11-09T10:15:36,323 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/41cd2f397b914cdcbf07109dd6458299, entries=150, sequenceid=59, filesize=11.7 K 2024-11-09T10:15:36,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/138c260325e14d2e889c1fde2682e0a5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/138c260325e14d2e889c1fde2682e0a5 2024-11-09T10:15:36,344 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/138c260325e14d2e889c1fde2682e0a5, entries=150, sequenceid=59, filesize=11.7 K 2024-11-09T10:15:36,350 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.55 KB/134705 for cac2483be99c556ef585845f10246658 in 644ms, sequenceid=59, compaction requested=true 2024-11-09T10:15:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-09T10:15:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-09T10:15:36,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T10:15:36,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-11-09T10:15:36,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9980 sec 2024-11-09T10:15:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T10:15:36,520 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:36,520 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-09T10:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T10:15:36,524 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:36,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:36,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:36,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=138.26 KB heapSize=394.23 KB 2024-11-09T10:15:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:36,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147335947, max timestamp=1731147336613Num uniques 1068; 2024-11-09T10:15:36,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1731147335947, max timestamp=1731147336614Num uniques 1068; 2024-11-09T10:15:36,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T10:15:36,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48900, new segment data size=7680, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T10:15:36,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:36,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:36,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:36,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:36,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:15:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T10:15:36,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/24f23a8790914f1cbbd488de8b9edfc8 is 50, key is test_row_0/A:col10/1731147336614/Put/seqid=0 2024-11-09T10:15:36,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741846_1022 (size=12001) 2024-11-09T10:15:36,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/24f23a8790914f1cbbd488de8b9edfc8 2024-11-09T10:15:36,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147396664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147396667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147396674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:36,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147396678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147396678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fea3d51582454e898ead0d291fc49f27 is 50, key is test_row_0/B:col10/1731147336614/Put/seqid=0 2024-11-09T10:15:36,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741847_1023 (size=12001) 2024-11-09T10:15:36,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147396789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147396789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147396789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147396799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147396800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:36,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:36,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T10:15:36,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:36,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147396999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147396999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147397000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147397020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147397019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,111 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T10:15:37,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fea3d51582454e898ead0d291fc49f27 2024-11-09T10:15:37,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4930b6a4b02e46a9bc939a22f52c6452 is 50, key is test_row_0/C:col10/1731147336614/Put/seqid=0 2024-11-09T10:15:37,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741848_1024 (size=12001) 2024-11-09T10:15:37,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T10:15:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:37,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147397310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147397310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147397310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147397329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147397329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:37,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:37,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48.80 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4930b6a4b02e46a9bc939a22f52c6452 2024-11-09T10:15:37,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/24f23a8790914f1cbbd488de8b9edfc8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/24f23a8790914f1cbbd488de8b9edfc8 2024-11-09T10:15:37,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/24f23a8790914f1cbbd488de8b9edfc8, entries=150, sequenceid=98, filesize=11.7 K 2024-11-09T10:15:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fea3d51582454e898ead0d291fc49f27 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fea3d51582454e898ead0d291fc49f27 2024-11-09T10:15:37,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fea3d51582454e898ead0d291fc49f27, entries=150, sequenceid=98, filesize=11.7 K 2024-11-09T10:15:37,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4930b6a4b02e46a9bc939a22f52c6452 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4930b6a4b02e46a9bc939a22f52c6452 2024-11-09T10:15:37,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:37,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4930b6a4b02e46a9bc939a22f52c6452, entries=150, sequenceid=98, filesize=11.7 K 2024-11-09T10:15:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72875, heapSize ~197.76 KB/202504, currentSize=107.34 KB/109920 for cac2483be99c556ef585845f10246658 in 1000ms, sequenceid=98, compaction requested=true 2024-11-09T10:15:37,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:37,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:37,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:37,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:37,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:37,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:15:37,625 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:37,626 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:37,631 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:37,631 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:37,633 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:37,634 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,634 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e720c0b80eba45648bd2a3d801a314b2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7843c7eaed2f4cd293d22a998d5f0921, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/41cd2f397b914cdcbf07109dd6458299, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fea3d51582454e898ead0d291fc49f27] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=46.9 K 2024-11-09T10:15:37,635 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:37,635 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,635 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/afa85605a5d54f9da7e9a9dc2e52a4dd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b54d28bc7e9a42248485df1265ee268a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6b9026ecbf042939f4e14b924ace161, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/24f23a8790914f1cbbd488de8b9edfc8] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=49.2 K 2024-11-09T10:15:37,636 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e720c0b80eba45648bd2a3d801a314b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731147334432 2024-11-09T10:15:37,641 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7843c7eaed2f4cd293d22a998d5f0921, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731147334726 2024-11-09T10:15:37,642 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting afa85605a5d54f9da7e9a9dc2e52a4dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731147334432 2024-11-09T10:15:37,643 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 41cd2f397b914cdcbf07109dd6458299, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147335453 2024-11-09T10:15:37,644 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b54d28bc7e9a42248485df1265ee268a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731147334726 2024-11-09T10:15:37,644 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting fea3d51582454e898ead0d291fc49f27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731147335951 2024-11-09T10:15:37,648 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b9026ecbf042939f4e14b924ace161, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147335453 2024-11-09T10:15:37,651 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f23a8790914f1cbbd488de8b9edfc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731147335951 2024-11-09T10:15:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T10:15:37,707 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#12 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:37,715 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/7116e9d9ca8d4e8a8d1ebde3b5bd50dd is 50, key is test_row_0/B:col10/1731147336614/Put/seqid=0 2024-11-09T10:15:37,715 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:37,717 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/04cc358d99d64e878bb784b64dd3bb57 is 50, key is test_row_0/A:col10/1731147336614/Put/seqid=0 2024-11-09T10:15:37,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741849_1025 (size=12139) 2024-11-09T10:15:37,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741850_1026 (size=12139) 2024-11-09T10:15:37,752 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/7116e9d9ca8d4e8a8d1ebde3b5bd50dd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7116e9d9ca8d4e8a8d1ebde3b5bd50dd 2024-11-09T10:15:37,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T10:15:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,771 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:15:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:37,781 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into 7116e9d9ca8d4e8a8d1ebde3b5bd50dd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:37,781 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:37,785 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=12, startTime=1731147337621; duration=0sec 2024-11-09T10:15:37,785 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:37,786 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:37,786 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:37,789 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:37,789 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/C is initiating minor compaction (all files) 2024-11-09T10:15:37,789 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/C in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:37,789 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2503f530f669445ab32be5438bb73adb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2b324bc37ada4624b022140725ae7843, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/138c260325e14d2e889c1fde2682e0a5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4930b6a4b02e46a9bc939a22f52c6452] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=46.9 K 2024-11-09T10:15:37,791 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2503f530f669445ab32be5438bb73adb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731147334432 2024-11-09T10:15:37,792 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b324bc37ada4624b022140725ae7843, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731147334726 2024-11-09T10:15:37,793 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 138c260325e14d2e889c1fde2682e0a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147335453 2024-11-09T10:15:37,794 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4930b6a4b02e46a9bc939a22f52c6452, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731147335951 2024-11-09T10:15:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:37,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/42e3620bc769459b9c9623534f4bd282 is 50, key is test_row_0/A:col10/1731147336663/Put/seqid=0 2024-11-09T10:15:37,873 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#C#compaction#15 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:37,874 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4932fc3ba5b6487a93d0fa8bdeac4a3c is 50, key is test_row_0/C:col10/1731147336614/Put/seqid=0 2024-11-09T10:15:37,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741851_1027 (size=12001) 2024-11-09T10:15:37,886 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/42e3620bc769459b9c9623534f4bd282 2024-11-09T10:15:37,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741852_1028 (size=12139) 2024-11-09T10:15:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/08f09161fc484b178a42a7792519f5e8 is 50, key is test_row_0/B:col10/1731147336663/Put/seqid=0 2024-11-09T10:15:37,910 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4932fc3ba5b6487a93d0fa8bdeac4a3c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4932fc3ba5b6487a93d0fa8bdeac4a3c 2024-11-09T10:15:37,925 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/C of cac2483be99c556ef585845f10246658 into 4932fc3ba5b6487a93d0fa8bdeac4a3c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:37,925 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:37,925 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/C, priority=12, startTime=1731147337621; duration=0sec 2024-11-09T10:15:37,925 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:37,926 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:37,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741853_1029 (size=12001) 2024-11-09T10:15:37,930 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/08f09161fc484b178a42a7792519f5e8 2024-11-09T10:15:37,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147397926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147397926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147397930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147397931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/c3b2e30c2fa6486d845105954d7378af is 50, key is test_row_0/C:col10/1731147336663/Put/seqid=0 2024-11-09T10:15:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147397934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:37,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741854_1030 (size=12001) 2024-11-09T10:15:37,969 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/c3b2e30c2fa6486d845105954d7378af 2024-11-09T10:15:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/42e3620bc769459b9c9623534f4bd282 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/42e3620bc769459b9c9623534f4bd282 2024-11-09T10:15:37,997 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/42e3620bc769459b9c9623534f4bd282, entries=150, sequenceid=117, filesize=11.7 K 2024-11-09T10:15:38,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/08f09161fc484b178a42a7792519f5e8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/08f09161fc484b178a42a7792519f5e8 2024-11-09T10:15:38,012 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/08f09161fc484b178a42a7792519f5e8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-09T10:15:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/c3b2e30c2fa6486d845105954d7378af as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/c3b2e30c2fa6486d845105954d7378af 2024-11-09T10:15:38,031 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/c3b2e30c2fa6486d845105954d7378af, entries=150, sequenceid=117, filesize=11.7 K 2024-11-09T10:15:38,033 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for cac2483be99c556ef585845f10246658 in 262ms, sequenceid=117, compaction requested=false 2024-11-09T10:15:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-09T10:15:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-09T10:15:38,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T10:15:38,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-11-09T10:15:38,045 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T10:15:38,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5220 sec 2024-11-09T10:15:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:38,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:38,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/d3253f37afbd446f90c854e642061abf is 50, key is test_row_0/A:col10/1731147338054/Put/seqid=0 2024-11-09T10:15:38,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741855_1031 (size=16881) 2024-11-09T10:15:38,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/d3253f37afbd446f90c854e642061abf 2024-11-09T10:15:38,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147398097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147398101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147398102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/6c3b631e2c6e4221ab155a9fa743c5c5 is 50, key is test_row_0/B:col10/1731147338054/Put/seqid=0 2024-11-09T10:15:38,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147398114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147398116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741856_1032 (size=12151) 2024-11-09T10:15:38,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/6c3b631e2c6e4221ab155a9fa743c5c5 2024-11-09T10:15:38,151 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/04cc358d99d64e878bb784b64dd3bb57 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/04cc358d99d64e878bb784b64dd3bb57 2024-11-09T10:15:38,166 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into 04cc358d99d64e878bb784b64dd3bb57(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:38,166 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:38,166 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=12, startTime=1731147337618; duration=0sec 2024-11-09T10:15:38,167 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:38,167 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:38,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/1e1cba6cf7464f1ca26d194b0a0d728a is 50, key is test_row_0/C:col10/1731147338054/Put/seqid=0 2024-11-09T10:15:38,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741857_1033 (size=12151) 2024-11-09T10:15:38,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/1e1cba6cf7464f1ca26d194b0a0d728a 2024-11-09T10:15:38,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/d3253f37afbd446f90c854e642061abf as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/d3253f37afbd446f90c854e642061abf 2024-11-09T10:15:38,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147398220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147398220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147398220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/d3253f37afbd446f90c854e642061abf, entries=250, sequenceid=135, filesize=16.5 K 2024-11-09T10:15:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/6c3b631e2c6e4221ab155a9fa743c5c5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/6c3b631e2c6e4221ab155a9fa743c5c5 2024-11-09T10:15:38,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147398239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/6c3b631e2c6e4221ab155a9fa743c5c5, entries=150, sequenceid=135, filesize=11.9 K 2024-11-09T10:15:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147398239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/1e1cba6cf7464f1ca26d194b0a0d728a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/1e1cba6cf7464f1ca26d194b0a0d728a 2024-11-09T10:15:38,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/1e1cba6cf7464f1ca26d194b0a0d728a, entries=150, sequenceid=135, filesize=11.9 K 2024-11-09T10:15:38,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for cac2483be99c556ef585845f10246658 in 206ms, sequenceid=135, compaction requested=true 2024-11-09T10:15:38,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:38,264 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:38,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:38,266 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:38,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:38,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:38,266 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:38,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:38,266 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:38,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:38,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:38,266 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,266 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/04cc358d99d64e878bb784b64dd3bb57, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/42e3620bc769459b9c9623534f4bd282, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/d3253f37afbd446f90c854e642061abf] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=40.1 K 2024-11-09T10:15:38,268 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04cc358d99d64e878bb784b64dd3bb57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731147335951 2024-11-09T10:15:38,268 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42e3620bc769459b9c9623534f4bd282, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731147336660 2024-11-09T10:15:38,269 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:38,269 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:38,269 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,270 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7116e9d9ca8d4e8a8d1ebde3b5bd50dd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/08f09161fc484b178a42a7792519f5e8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/6c3b631e2c6e4221ab155a9fa743c5c5] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=35.4 K 2024-11-09T10:15:38,270 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3253f37afbd446f90c854e642061abf, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731147337882 2024-11-09T10:15:38,270 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7116e9d9ca8d4e8a8d1ebde3b5bd50dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731147335951 2024-11-09T10:15:38,272 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 08f09161fc484b178a42a7792519f5e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731147336660 2024-11-09T10:15:38,272 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3b631e2c6e4221ab155a9fa743c5c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731147337929 2024-11-09T10:15:38,300 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:38,301 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/89c19d6b21144f31a1116624e11b0f4f is 50, key is test_row_0/A:col10/1731147338054/Put/seqid=0 2024-11-09T10:15:38,307 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T10:15:38,307 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T10:15:38,310 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T10:15:38,311 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T10:15:38,311 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T10:15:38,311 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T10:15:38,313 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T10:15:38,313 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T10:15:38,324 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:38,325 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/2f1328129dea4842a754102e435a2f77 is 50, key is test_row_0/B:col10/1731147338054/Put/seqid=0 2024-11-09T10:15:38,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741859_1035 (size=12391) 2024-11-09T10:15:38,360 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/2f1328129dea4842a754102e435a2f77 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/2f1328129dea4842a754102e435a2f77 2024-11-09T10:15:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741858_1034 (size=12391) 2024-11-09T10:15:38,377 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into 2f1328129dea4842a754102e435a2f77(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:38,377 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:38,377 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=13, startTime=1731147338266; duration=0sec 2024-11-09T10:15:38,377 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:38,378 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:38,378 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:38,379 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/89c19d6b21144f31a1116624e11b0f4f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/89c19d6b21144f31a1116624e11b0f4f 2024-11-09T10:15:38,380 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:38,380 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/C is initiating minor compaction (all files) 2024-11-09T10:15:38,380 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/C in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,380 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4932fc3ba5b6487a93d0fa8bdeac4a3c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/c3b2e30c2fa6486d845105954d7378af, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/1e1cba6cf7464f1ca26d194b0a0d728a] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=35.4 K 2024-11-09T10:15:38,381 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4932fc3ba5b6487a93d0fa8bdeac4a3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731147335951 2024-11-09T10:15:38,383 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b2e30c2fa6486d845105954d7378af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731147336660 2024-11-09T10:15:38,386 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e1cba6cf7464f1ca26d194b0a0d728a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731147337929 2024-11-09T10:15:38,397 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into 89c19d6b21144f31a1116624e11b0f4f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:38,397 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:38,397 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=13, startTime=1731147338263; duration=0sec 2024-11-09T10:15:38,398 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:38,398 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:38,407 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:38,408 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/dee9805bad22450ba2e9d6486ff44766 is 50, key is test_row_0/C:col10/1731147338054/Put/seqid=0 2024-11-09T10:15:38,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741860_1036 (size=12391) 2024-11-09T10:15:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:38,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T10:15:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:38,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/9217a9e17e724983bc1037e90f7db46a is 50, key is test_row_0/A:col10/1731147338112/Put/seqid=0 2024-11-09T10:15:38,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741861_1037 (size=14541) 2024-11-09T10:15:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147398517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147398523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147398529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147398531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147398532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147398640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147398640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147398649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147398659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147398660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T10:15:38,671 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:38,672 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-09T10:15:38,675 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T10:15:38,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:38,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T10:15:38,830 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/dee9805bad22450ba2e9d6486ff44766 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/dee9805bad22450ba2e9d6486ff44766 2024-11-09T10:15:38,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T10:15:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:38,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:38,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:38,853 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/C of cac2483be99c556ef585845f10246658 into dee9805bad22450ba2e9d6486ff44766(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:38,853 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:38,853 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/C, priority=13, startTime=1731147338266; duration=0sec 2024-11-09T10:15:38,853 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:38,853 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147398850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147398850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147398859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/9217a9e17e724983bc1037e90f7db46a 2024-11-09T10:15:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147398870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147398870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:38,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/dd3ae459b6494d1e815fb5db1d30d243 is 50, key is test_row_0/B:col10/1731147338112/Put/seqid=0 2024-11-09T10:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:38,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741862_1038 (size=12151) 2024-11-09T10:15:38,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T10:15:38,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:38,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:38,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:38,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T10:15:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:39,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T10:15:39,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:39,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:39,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:39,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:39,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:39,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147399169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147399173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147399173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147399189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147399190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/dd3ae459b6494d1e815fb5db1d30d243 2024-11-09T10:15:39,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/20f4205494f842a29851773e167438e1 is 50, key is test_row_0/C:col10/1731147338112/Put/seqid=0 2024-11-09T10:15:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T10:15:39,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T10:15:39,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741863_1039 (size=12151) 2024-11-09T10:15:39,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:39,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:39,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:39,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:39,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:39,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/20f4205494f842a29851773e167438e1 2024-11-09T10:15:39,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/9217a9e17e724983bc1037e90f7db46a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/9217a9e17e724983bc1037e90f7db46a 2024-11-09T10:15:39,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/9217a9e17e724983bc1037e90f7db46a, entries=200, sequenceid=156, filesize=14.2 K 2024-11-09T10:15:39,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/dd3ae459b6494d1e815fb5db1d30d243 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/dd3ae459b6494d1e815fb5db1d30d243 2024-11-09T10:15:39,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/dd3ae459b6494d1e815fb5db1d30d243, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T10:15:39,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/20f4205494f842a29851773e167438e1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/20f4205494f842a29851773e167438e1 2024-11-09T10:15:39,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/20f4205494f842a29851773e167438e1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T10:15:39,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cac2483be99c556ef585845f10246658 in 947ms, sequenceid=156, compaction requested=false 2024-11-09T10:15:39,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:39,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T10:15:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:39,465 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T10:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:39,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/30083a02179146eebe45f5657e9963b1 is 50, key is test_row_0/A:col10/1731147338509/Put/seqid=0 2024-11-09T10:15:39,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741864_1040 (size=12151) 2024-11-09T10:15:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:39,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:39,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147399744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147399744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147399747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147399750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147399753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T10:15:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147399859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147399859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147399869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147399869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147399869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:39,895 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/30083a02179146eebe45f5657e9963b1 2024-11-09T10:15:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bd87185b5ed5401ea9b071782dbb73d1 is 50, key is test_row_0/B:col10/1731147338509/Put/seqid=0 2024-11-09T10:15:39,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741865_1041 (size=12151) 2024-11-09T10:15:39,963 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bd87185b5ed5401ea9b071782dbb73d1 2024-11-09T10:15:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/89c315d57e74439c8acf1755eecaf44e is 50, key is test_row_0/C:col10/1731147338509/Put/seqid=0 2024-11-09T10:15:40,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741866_1042 (size=12151) 2024-11-09T10:15:40,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147400069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147400069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147400080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147400080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147400080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147400379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147400379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147400389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147400389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147400389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,413 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/89c315d57e74439c8acf1755eecaf44e 2024-11-09T10:15:40,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/30083a02179146eebe45f5657e9963b1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/30083a02179146eebe45f5657e9963b1 2024-11-09T10:15:40,430 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/30083a02179146eebe45f5657e9963b1, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T10:15:40,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bd87185b5ed5401ea9b071782dbb73d1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd87185b5ed5401ea9b071782dbb73d1 2024-11-09T10:15:40,446 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd87185b5ed5401ea9b071782dbb73d1, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T10:15:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/89c315d57e74439c8acf1755eecaf44e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/89c315d57e74439c8acf1755eecaf44e 2024-11-09T10:15:40,466 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/89c315d57e74439c8acf1755eecaf44e, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T10:15:40,468 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cac2483be99c556ef585845f10246658 in 1004ms, sequenceid=171, compaction requested=true 2024-11-09T10:15:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-09T10:15:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-09T10:15:40,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T10:15:40,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-11-09T10:15:40,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8090 sec 2024-11-09T10:15:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T10:15:40,840 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:40,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-09T10:15:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T10:15:40,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:40,846 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:40,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:40,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:15:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:40,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e6652dd73acb471ab83d1a07c31ecb0b is 50, key is test_row_0/A:col10/1731147339720/Put/seqid=0 2024-11-09T10:15:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147400913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147400913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147400915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147400915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147400919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741867_1043 (size=14541) 2024-11-09T10:15:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T10:15:41,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T10:15:41,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:41,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147401029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147401029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147401039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147401039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147401039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T10:15:41,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:41,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T10:15:41,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147401239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147401239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147401249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147401250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147401250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T10:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e6652dd73acb471ab83d1a07c31ecb0b 2024-11-09T10:15:41,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bd2d22b9c7d641e4abb84bf956c63c12 is 50, key is test_row_0/B:col10/1731147339720/Put/seqid=0 2024-11-09T10:15:41,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741868_1044 (size=12151) 2024-11-09T10:15:41,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bd2d22b9c7d641e4abb84bf956c63c12 2024-11-09T10:15:41,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/5aab89a7402a4eaaa02a65e46e2007bc is 50, key is test_row_0/C:col10/1731147339720/Put/seqid=0 2024-11-09T10:15:41,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741869_1045 (size=12151) 2024-11-09T10:15:41,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T10:15:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T10:15:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147401549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147401549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147401559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147401559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147401559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:41,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T10:15:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T10:15:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:41,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:41,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/5aab89a7402a4eaaa02a65e46e2007bc 2024-11-09T10:15:41,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e6652dd73acb471ab83d1a07c31ecb0b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6652dd73acb471ab83d1a07c31ecb0b 2024-11-09T10:15:41,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6652dd73acb471ab83d1a07c31ecb0b, entries=200, sequenceid=190, filesize=14.2 K 2024-11-09T10:15:41,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bd2d22b9c7d641e4abb84bf956c63c12 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd2d22b9c7d641e4abb84bf956c63c12 2024-11-09T10:15:41,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd2d22b9c7d641e4abb84bf956c63c12, entries=150, sequenceid=190, filesize=11.9 K 2024-11-09T10:15:41,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/5aab89a7402a4eaaa02a65e46e2007bc as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/5aab89a7402a4eaaa02a65e46e2007bc 2024-11-09T10:15:41,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/5aab89a7402a4eaaa02a65e46e2007bc, entries=150, sequenceid=190, filesize=11.9 K 2024-11-09T10:15:41,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cac2483be99c556ef585845f10246658 in 951ms, sequenceid=190, compaction requested=true 2024-11-09T10:15:41,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:41,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:41,851 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:41,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:41,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:41,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:41,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:41,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:41,852 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:41,854 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:41,854 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:41,854 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,854 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/2f1328129dea4842a754102e435a2f77, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/dd3ae459b6494d1e815fb5db1d30d243, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd87185b5ed5401ea9b071782dbb73d1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd2d22b9c7d641e4abb84bf956c63c12] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=47.7 K 2024-11-09T10:15:41,855 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53624 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:41,855 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:41,855 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,855 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/89c19d6b21144f31a1116624e11b0f4f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/9217a9e17e724983bc1037e90f7db46a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/30083a02179146eebe45f5657e9963b1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6652dd73acb471ab83d1a07c31ecb0b] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=52.4 K 2024-11-09T10:15:41,855 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f1328129dea4842a754102e435a2f77, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731147337929 2024-11-09T10:15:41,856 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c19d6b21144f31a1116624e11b0f4f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731147337929 2024-11-09T10:15:41,856 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting dd3ae459b6494d1e815fb5db1d30d243, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147338090 2024-11-09T10:15:41,857 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9217a9e17e724983bc1037e90f7db46a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147338090 2024-11-09T10:15:41,857 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bd87185b5ed5401ea9b071782dbb73d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731147338509 2024-11-09T10:15:41,858 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30083a02179146eebe45f5657e9963b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731147338509 2024-11-09T10:15:41,858 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6652dd73acb471ab83d1a07c31ecb0b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731147339720 2024-11-09T10:15:41,859 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2d22b9c7d641e4abb84bf956c63c12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731147339720 2024-11-09T10:15:41,881 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#33 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:41,883 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/3b5f7dbcab494d76b0e52f674e298ea6 is 50, key is test_row_0/A:col10/1731147339720/Put/seqid=0 2024-11-09T10:15:41,891 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:41,892 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/5eafdaa93e4d4f8e870acb3ea56fd4d7 is 50, key is test_row_0/B:col10/1731147339720/Put/seqid=0 2024-11-09T10:15:41,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741870_1046 (size=12527) 2024-11-09T10:15:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741871_1047 (size=12527) 2024-11-09T10:15:41,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T10:15:41,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,926 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:15:41,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:41,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:41,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:41,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:41,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:41,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/68898c7aaf024c0a94ecc025914ba907 is 50, key is test_row_0/A:col10/1731147340918/Put/seqid=0 2024-11-09T10:15:41,939 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/5eafdaa93e4d4f8e870acb3ea56fd4d7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5eafdaa93e4d4f8e870acb3ea56fd4d7 2024-11-09T10:15:41,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741872_1048 (size=12151) 2024-11-09T10:15:41,949 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/68898c7aaf024c0a94ecc025914ba907 2024-11-09T10:15:41,950 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into 5eafdaa93e4d4f8e870acb3ea56fd4d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:41,951 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:41,951 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=12, startTime=1731147341851; duration=0sec 2024-11-09T10:15:41,951 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:41,951 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:41,951 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:41,957 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:41,957 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/C is initiating minor compaction (all files) 2024-11-09T10:15:41,958 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/C in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:41,958 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/dee9805bad22450ba2e9d6486ff44766, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/20f4205494f842a29851773e167438e1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/89c315d57e74439c8acf1755eecaf44e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/5aab89a7402a4eaaa02a65e46e2007bc] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=47.7 K 2024-11-09T10:15:41,959 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting dee9805bad22450ba2e9d6486ff44766, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731147337929 2024-11-09T10:15:41,959 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f4205494f842a29851773e167438e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147338090 2024-11-09T10:15:41,960 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 89c315d57e74439c8acf1755eecaf44e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731147338509 2024-11-09T10:15:41,961 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aab89a7402a4eaaa02a65e46e2007bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731147339720 2024-11-09T10:15:41,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/ce13743aa6414c72b87c33ba4c9b9e76 is 50, key is test_row_0/B:col10/1731147340918/Put/seqid=0 2024-11-09T10:15:41,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741873_1049 (size=12151) 2024-11-09T10:15:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T10:15:41,987 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:41,988 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4f3fcad3c62d45b689fc391d09f9306d is 50, key is test_row_0/C:col10/1731147339720/Put/seqid=0 2024-11-09T10:15:41,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741874_1050 (size=12527) 2024-11-09T10:15:42,005 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4f3fcad3c62d45b689fc391d09f9306d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4f3fcad3c62d45b689fc391d09f9306d 2024-11-09T10:15:42,018 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/C of cac2483be99c556ef585845f10246658 into 4f3fcad3c62d45b689fc391d09f9306d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:42,018 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:42,018 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/C, priority=12, startTime=1731147341852; duration=0sec 2024-11-09T10:15:42,018 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:42,018 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:42,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:42,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147402093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147402094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147402095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147402096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147402099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147402208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147402209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147402209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147402209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147402209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,328 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/3b5f7dbcab494d76b0e52f674e298ea6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/3b5f7dbcab494d76b0e52f674e298ea6 2024-11-09T10:15:42,337 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into 3b5f7dbcab494d76b0e52f674e298ea6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:42,338 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:42,338 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=12, startTime=1731147341851; duration=0sec 2024-11-09T10:15:42,338 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:42,338 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:42,377 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/ce13743aa6414c72b87c33ba4c9b9e76 2024-11-09T10:15:42,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4a5e8e9117954dbdb0da0486cb2beea2 is 50, key is test_row_0/C:col10/1731147340918/Put/seqid=0 2024-11-09T10:15:42,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741875_1051 (size=12151) 2024-11-09T10:15:42,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147402419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147402419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147402419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147402419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147402419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147402729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147402729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147402729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147402729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147402739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:42,808 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4a5e8e9117954dbdb0da0486cb2beea2 2024-11-09T10:15:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/68898c7aaf024c0a94ecc025914ba907 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/68898c7aaf024c0a94ecc025914ba907 2024-11-09T10:15:42,863 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/68898c7aaf024c0a94ecc025914ba907, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T10:15:42,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/ce13743aa6414c72b87c33ba4c9b9e76 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ce13743aa6414c72b87c33ba4c9b9e76 2024-11-09T10:15:42,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ce13743aa6414c72b87c33ba4c9b9e76, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T10:15:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4a5e8e9117954dbdb0da0486cb2beea2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4a5e8e9117954dbdb0da0486cb2beea2 2024-11-09T10:15:42,887 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4a5e8e9117954dbdb0da0486cb2beea2, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T10:15:42,888 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cac2483be99c556ef585845f10246658 in 962ms, sequenceid=203, compaction requested=false 2024-11-09T10:15:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-09T10:15:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-09T10:15:42,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T10:15:42,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0440 sec 2024-11-09T10:15:42,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0560 sec 2024-11-09T10:15:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T10:15:42,989 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:42,989 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-09T10:15:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T10:15:42,998 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:43,000 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:43,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T10:15:43,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T10:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:43,155 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b0cdae342e2d42c09061093d7736e239 is 50, key is test_row_0/A:col10/1731147342094/Put/seqid=0 2024-11-09T10:15:43,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741876_1052 (size=12151) 2024-11-09T10:15:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147403255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147403255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147403256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147403262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147403262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T10:15:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147403369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147403369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147403369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147403379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147403379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,574 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b0cdae342e2d42c09061093d7736e239 2024-11-09T10:15:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147403579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147403579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/961262bcb03c434c8e319ac7cfb2a35e is 50, key is test_row_0/B:col10/1731147342094/Put/seqid=0 2024-11-09T10:15:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147403579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741877_1053 (size=12151) 2024-11-09T10:15:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147403589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147403590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,591 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/961262bcb03c434c8e319ac7cfb2a35e 2024-11-09T10:15:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/3fb4da58b52b4b779b91eec8a31dd6ce is 50, key is test_row_0/C:col10/1731147342094/Put/seqid=0 2024-11-09T10:15:43,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741878_1054 (size=12151) 2024-11-09T10:15:43,611 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/3fb4da58b52b4b779b91eec8a31dd6ce 2024-11-09T10:15:43,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b0cdae342e2d42c09061093d7736e239 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b0cdae342e2d42c09061093d7736e239 2024-11-09T10:15:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T10:15:43,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b0cdae342e2d42c09061093d7736e239, entries=150, sequenceid=225, filesize=11.9 K 2024-11-09T10:15:43,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/961262bcb03c434c8e319ac7cfb2a35e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/961262bcb03c434c8e319ac7cfb2a35e 2024-11-09T10:15:43,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/961262bcb03c434c8e319ac7cfb2a35e, entries=150, sequenceid=225, filesize=11.9 K 2024-11-09T10:15:43,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/3fb4da58b52b4b779b91eec8a31dd6ce as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3fb4da58b52b4b779b91eec8a31dd6ce 2024-11-09T10:15:43,644 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3fb4da58b52b4b779b91eec8a31dd6ce, entries=150, sequenceid=225, filesize=11.9 K 2024-11-09T10:15:43,645 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for cac2483be99c556ef585845f10246658 in 490ms, sequenceid=225, compaction requested=true 2024-11-09T10:15:43,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:43,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:43,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-09T10:15:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-09T10:15:43,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T10:15:43,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 648 msec 2024-11-09T10:15:43,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 661 msec 2024-11-09T10:15:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:43,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:15:43,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:43,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:43,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:43,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:43,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:43,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:43,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/55d1d230a1d84e2596e4b79ca7c3f258 is 50, key is test_row_0/A:col10/1731147343893/Put/seqid=0 2024-11-09T10:15:43,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741879_1055 (size=16931) 2024-11-09T10:15:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147403914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147403916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147403918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147403921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:43,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147403921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:43,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:43,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147404029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147404029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147404029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147404029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147404039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T10:15:44,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:44,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-09T10:15:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T10:15:44,132 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:44,133 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:44,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T10:15:44,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147404238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147404239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147404239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147404249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147404249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T10:15:44,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/55d1d230a1d84e2596e4b79ca7c3f258 2024-11-09T10:15:44,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fe50d42495214442a59b3e7591335f17 is 50, key is test_row_0/B:col10/1731147343893/Put/seqid=0 2024-11-09T10:15:44,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741880_1056 (size=12151) 2024-11-09T10:15:44,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T10:15:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T10:15:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147404548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147404549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147404550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147404559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147404559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:44,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T10:15:44,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:44,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fe50d42495214442a59b3e7591335f17 2024-11-09T10:15:44,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/9dfa35bc3e7043628843d3cfffe9c2d8 is 50, key is test_row_0/C:col10/1731147343893/Put/seqid=0 2024-11-09T10:15:44,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T10:15:44,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:44,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T10:15:44,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741881_1057 (size=12151) 2024-11-09T10:15:44,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T10:15:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:44,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:45,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T10:15:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:45,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147405058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147405059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147405059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:45,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147405069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147405078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:45,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/9dfa35bc3e7043628843d3cfffe9c2d8 2024-11-09T10:15:45,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/55d1d230a1d84e2596e4b79ca7c3f258 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/55d1d230a1d84e2596e4b79ca7c3f258 2024-11-09T10:15:45,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/55d1d230a1d84e2596e4b79ca7c3f258, entries=250, sequenceid=241, filesize=16.5 K 2024-11-09T10:15:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fe50d42495214442a59b3e7591335f17 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fe50d42495214442a59b3e7591335f17 2024-11-09T10:15:45,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fe50d42495214442a59b3e7591335f17, entries=150, sequenceid=241, filesize=11.9 K 2024-11-09T10:15:45,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/9dfa35bc3e7043628843d3cfffe9c2d8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/9dfa35bc3e7043628843d3cfffe9c2d8 2024-11-09T10:15:45,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/9dfa35bc3e7043628843d3cfffe9c2d8, entries=150, sequenceid=241, filesize=11.9 K 2024-11-09T10:15:45,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for cac2483be99c556ef585845f10246658 in 1310ms, sequenceid=241, compaction requested=true 2024-11-09T10:15:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:45,204 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:45,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:45,206 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:45,206 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:45,206 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:45,207 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:45,207 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/3b5f7dbcab494d76b0e52f674e298ea6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/68898c7aaf024c0a94ecc025914ba907, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b0cdae342e2d42c09061093d7736e239, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/55d1d230a1d84e2596e4b79ca7c3f258] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=52.5 K 2024-11-09T10:15:45,207 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b5f7dbcab494d76b0e52f674e298ea6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731147339720 2024-11-09T10:15:45,208 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:45,208 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:45,208 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:45,208 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5eafdaa93e4d4f8e870acb3ea56fd4d7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ce13743aa6414c72b87c33ba4c9b9e76, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/961262bcb03c434c8e319ac7cfb2a35e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fe50d42495214442a59b3e7591335f17] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=47.8 K 2024-11-09T10:15:45,209 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68898c7aaf024c0a94ecc025914ba907, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731147340912 2024-11-09T10:15:45,210 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eafdaa93e4d4f8e870acb3ea56fd4d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731147339720 2024-11-09T10:15:45,211 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting ce13743aa6414c72b87c33ba4c9b9e76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731147340912 2024-11-09T10:15:45,211 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0cdae342e2d42c09061093d7736e239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731147342073 2024-11-09T10:15:45,212 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 961262bcb03c434c8e319ac7cfb2a35e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731147342073 2024-11-09T10:15:45,212 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55d1d230a1d84e2596e4b79ca7c3f258, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731147343245 2024-11-09T10:15:45,213 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting fe50d42495214442a59b3e7591335f17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731147343257 2024-11-09T10:15:45,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T10:15:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:45,215 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T10:15:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:45,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:45,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:45,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:45,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:45,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:45,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:45,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b94dcf27b1ed45519c92821f8abccee6 is 50, key is test_row_0/A:col10/1731147343918/Put/seqid=0 2024-11-09T10:15:45,238 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#46 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:45,238 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:45,238 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/edba07735dbf433b8beb99d802339499 is 50, key is test_row_0/B:col10/1731147343893/Put/seqid=0 2024-11-09T10:15:45,239 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/2e485bfed72e4176b54bcd54d1abc0bb is 50, key is test_row_0/A:col10/1731147343893/Put/seqid=0 2024-11-09T10:15:45,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741882_1058 (size=12663) 2024-11-09T10:15:45,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741883_1059 (size=12663) 2024-11-09T10:15:45,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741884_1060 (size=12201) 2024-11-09T10:15:45,259 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b94dcf27b1ed45519c92821f8abccee6 2024-11-09T10:15:45,263 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/2e485bfed72e4176b54bcd54d1abc0bb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/2e485bfed72e4176b54bcd54d1abc0bb 2024-11-09T10:15:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T10:15:45,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/3973e5d2c69143048e8861182934d78a is 50, key is test_row_0/B:col10/1731147343918/Put/seqid=0 2024-11-09T10:15:45,277 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into 2e485bfed72e4176b54bcd54d1abc0bb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:45,277 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:45,277 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=12, startTime=1731147345204; duration=0sec 2024-11-09T10:15:45,277 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:45,277 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:45,278 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:45,279 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:45,279 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/C is initiating minor compaction (all files) 2024-11-09T10:15:45,279 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/C in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:45,279 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4f3fcad3c62d45b689fc391d09f9306d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4a5e8e9117954dbdb0da0486cb2beea2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3fb4da58b52b4b779b91eec8a31dd6ce, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/9dfa35bc3e7043628843d3cfffe9c2d8] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=47.8 K 2024-11-09T10:15:45,279 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f3fcad3c62d45b689fc391d09f9306d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731147339720 2024-11-09T10:15:45,280 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5e8e9117954dbdb0da0486cb2beea2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731147340912 2024-11-09T10:15:45,280 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fb4da58b52b4b779b91eec8a31dd6ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731147342073 2024-11-09T10:15:45,280 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dfa35bc3e7043628843d3cfffe9c2d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731147343257 2024-11-09T10:15:45,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741885_1061 (size=12201) 2024-11-09T10:15:45,295 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#C#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:45,296 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/07281b71a0854bb8aa331162ff292bc5 is 50, key is test_row_0/C:col10/1731147343893/Put/seqid=0 2024-11-09T10:15:45,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741886_1062 (size=12663) 2024-11-09T10:15:45,665 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/edba07735dbf433b8beb99d802339499 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/edba07735dbf433b8beb99d802339499 2024-11-09T10:15:45,673 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into edba07735dbf433b8beb99d802339499(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:45,673 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:45,673 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=12, startTime=1731147345206; duration=0sec 2024-11-09T10:15:45,673 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:45,673 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:45,683 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/3973e5d2c69143048e8861182934d78a 2024-11-09T10:15:45,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/6354a920550f471aa8d7134d12c47941 is 50, key is test_row_0/C:col10/1731147343918/Put/seqid=0 2024-11-09T10:15:45,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741887_1063 (size=12201) 2024-11-09T10:15:45,698 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/6354a920550f471aa8d7134d12c47941 2024-11-09T10:15:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/b94dcf27b1ed45519c92821f8abccee6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b94dcf27b1ed45519c92821f8abccee6 2024-11-09T10:15:45,709 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/07281b71a0854bb8aa331162ff292bc5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/07281b71a0854bb8aa331162ff292bc5 2024-11-09T10:15:45,718 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b94dcf27b1ed45519c92821f8abccee6, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T10:15:45,719 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/C of cac2483be99c556ef585845f10246658 into 07281b71a0854bb8aa331162ff292bc5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:45,719 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:45,719 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/C, priority=12, startTime=1731147345222; duration=0sec 2024-11-09T10:15:45,719 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:45,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/3973e5d2c69143048e8861182934d78a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/3973e5d2c69143048e8861182934d78a 2024-11-09T10:15:45,719 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:45,726 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/3973e5d2c69143048e8861182934d78a, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T10:15:45,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/6354a920550f471aa8d7134d12c47941 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/6354a920550f471aa8d7134d12c47941 2024-11-09T10:15:45,738 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/6354a920550f471aa8d7134d12c47941, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T10:15:45,739 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for cac2483be99c556ef585845f10246658 in 524ms, sequenceid=258, compaction requested=false 2024-11-09T10:15:45,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:45,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:45,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-09T10:15:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-09T10:15:45,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T10:15:45,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-09T10:15:45,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6170 sec 2024-11-09T10:15:46,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:15:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:46,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/c3a2d98d184643be8dd350ec15063f6e is 50, key is test_row_0/A:col10/1731147346095/Put/seqid=0 2024-11-09T10:15:46,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741888_1064 (size=9857) 2024-11-09T10:15:46,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147346128, max timestamp=1731147346161Num uniques 859; 2024-11-09T10:15:46,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:15:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147406150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147346128, max timestamp=1731147346161Num uniques 859; 2024-11-09T10:15:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:15:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731147346128, max timestamp=1731147346160Num uniques 859; 2024-11-09T10:15:46,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T10:15:46,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147406213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147406217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147406226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147406226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147406269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T10:15:46,279 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:46,280 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-09T10:15:46,283 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T10:15:46,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:46,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147406339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147406339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147406349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147406349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T10:15:46,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147406479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147406549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/c3a2d98d184643be8dd350ec15063f6e 2024-11-09T10:15:46,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147406550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147406569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147406569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/1d3fc42f975d409a851b27bac975998f is 50, key is test_row_0/B:col10/1731147346095/Put/seqid=0 2024-11-09T10:15:46,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:46,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:46,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T10:15:46,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741889_1065 (size=9857) 2024-11-09T10:15:46,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147406789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147406858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147406858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147406879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147406879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:46,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:46,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T10:15:47,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/1d3fc42f975d409a851b27bac975998f 2024-11-09T10:15:47,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/21d8048de5004dcda82b0b9d453b8bfe is 50, key is test_row_0/C:col10/1731147346095/Put/seqid=0 2024-11-09T10:15:47,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741890_1066 (size=9857) 2024-11-09T10:15:47,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147407299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:47,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147407369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147407369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:47,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147407389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147407389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T10:15:47,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/21d8048de5004dcda82b0b9d453b8bfe 2024-11-09T10:15:47,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/c3a2d98d184643be8dd350ec15063f6e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c3a2d98d184643be8dd350ec15063f6e 2024-11-09T10:15:47,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c3a2d98d184643be8dd350ec15063f6e, entries=100, sequenceid=273, filesize=9.6 K 2024-11-09T10:15:47,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/1d3fc42f975d409a851b27bac975998f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/1d3fc42f975d409a851b27bac975998f 2024-11-09T10:15:47,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/1d3fc42f975d409a851b27bac975998f, entries=100, sequenceid=273, filesize=9.6 K 2024-11-09T10:15:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/21d8048de5004dcda82b0b9d453b8bfe as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21d8048de5004dcda82b0b9d453b8bfe 2024-11-09T10:15:47,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21d8048de5004dcda82b0b9d453b8bfe, entries=100, sequenceid=273, filesize=9.6 K 2024-11-09T10:15:47,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=124.84 KB/127835 for cac2483be99c556ef585845f10246658 in 1397ms, sequenceid=273, compaction requested=true 2024-11-09T10:15:47,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:47,493 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:47,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:47,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:47,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:47,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:47,494 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:47,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:47,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:47,495 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:47,495 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:47,495 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,495 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/2e485bfed72e4176b54bcd54d1abc0bb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b94dcf27b1ed45519c92821f8abccee6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c3a2d98d184643be8dd350ec15063f6e] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=33.9 K 2024-11-09T10:15:47,496 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:47,496 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e485bfed72e4176b54bcd54d1abc0bb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731147343257 2024-11-09T10:15:47,496 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:47,496 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,496 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/edba07735dbf433b8beb99d802339499, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/3973e5d2c69143048e8861182934d78a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/1d3fc42f975d409a851b27bac975998f] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=33.9 K 2024-11-09T10:15:47,496 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b94dcf27b1ed45519c92821f8abccee6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147343915 2024-11-09T10:15:47,497 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting edba07735dbf433b8beb99d802339499, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731147343257 2024-11-09T10:15:47,497 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a2d98d184643be8dd350ec15063f6e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731147346091 2024-11-09T10:15:47,498 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 3973e5d2c69143048e8861182934d78a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147343915 2024-11-09T10:15:47,498 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d3fc42f975d409a851b27bac975998f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731147346091 2024-11-09T10:15:47,514 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:47,515 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e0014759961c4482ad5533698fcff9d9 is 50, key is test_row_0/A:col10/1731147346095/Put/seqid=0 2024-11-09T10:15:47,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T10:15:47,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,526 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=124.84 KB heapSize=354.38 KB 2024-11-09T10:15:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:47,528 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:47,528 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/80ebb97782004dfca0beb11d53934b86 is 50, key is test_row_0/B:col10/1731147346095/Put/seqid=0 2024-11-09T10:15:47,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/6103be6e27394b9daf75a7413c0d0ea9 is 50, key is test_row_0/A:col10/1731147346203/Put/seqid=0 2024-11-09T10:15:47,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741891_1067 (size=12865) 2024-11-09T10:15:47,558 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/e0014759961c4482ad5533698fcff9d9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e0014759961c4482ad5533698fcff9d9 2024-11-09T10:15:47,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741892_1068 (size=12865) 2024-11-09T10:15:47,570 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into e0014759961c4482ad5533698fcff9d9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:47,570 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:47,571 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=13, startTime=1731147347493; duration=0sec 2024-11-09T10:15:47,571 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:47,571 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:47,571 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:47,572 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/80ebb97782004dfca0beb11d53934b86 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/80ebb97782004dfca0beb11d53934b86 2024-11-09T10:15:47,572 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:47,573 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/C is initiating minor compaction (all files) 2024-11-09T10:15:47,573 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/C in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:47,573 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/07281b71a0854bb8aa331162ff292bc5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/6354a920550f471aa8d7134d12c47941, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21d8048de5004dcda82b0b9d453b8bfe] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=33.9 K 2024-11-09T10:15:47,573 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07281b71a0854bb8aa331162ff292bc5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731147343257 2024-11-09T10:15:47,574 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6354a920550f471aa8d7134d12c47941, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147343915 2024-11-09T10:15:47,574 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d8048de5004dcda82b0b9d453b8bfe, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731147346091 2024-11-09T10:15:47,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741893_1069 (size=12301) 2024-11-09T10:15:47,586 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/6103be6e27394b9daf75a7413c0d0ea9 2024-11-09T10:15:47,587 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into 80ebb97782004dfca0beb11d53934b86(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:47,587 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:47,587 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=13, startTime=1731147347493; duration=0sec 2024-11-09T10:15:47,589 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#C#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:47,589 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:47,589 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:47,590 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/ea3cfa5dc7694248a4f53bea47a974c5 is 50, key is test_row_0/C:col10/1731147346095/Put/seqid=0 2024-11-09T10:15:47,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/e03ee48e0e5044729de758da0c227c56 is 50, key is test_row_0/B:col10/1731147346203/Put/seqid=0 2024-11-09T10:15:47,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741894_1070 (size=12865) 2024-11-09T10:15:47,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741895_1071 (size=12301) 2024-11-09T10:15:48,012 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/ea3cfa5dc7694248a4f53bea47a974c5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/ea3cfa5dc7694248a4f53bea47a974c5 2024-11-09T10:15:48,014 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/e03ee48e0e5044729de758da0c227c56 2024-11-09T10:15:48,018 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/C of cac2483be99c556ef585845f10246658 into ea3cfa5dc7694248a4f53bea47a974c5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:48,018 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:48,018 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/C, priority=13, startTime=1731147347494; duration=0sec 2024-11-09T10:15:48,019 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:48,019 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/3cc56ace0f80431cac4a38809f6a55cc is 50, key is test_row_0/C:col10/1731147346203/Put/seqid=0 2024-11-09T10:15:48,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741896_1072 (size=12301) 2024-11-09T10:15:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:48,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:48,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147408342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:48,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147408379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,384 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:48,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147408389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,390 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,412 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:48,414 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:48,428 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/3cc56ace0f80431cac4a38809f6a55cc 2024-11-09T10:15:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/6103be6e27394b9daf75a7413c0d0ea9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6103be6e27394b9daf75a7413c0d0ea9 2024-11-09T10:15:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T10:15:48,446 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6103be6e27394b9daf75a7413c0d0ea9, entries=150, sequenceid=308, filesize=12.0 K 2024-11-09T10:15:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/e03ee48e0e5044729de758da0c227c56 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e03ee48e0e5044729de758da0c227c56 2024-11-09T10:15:48,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147408449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:48,459 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e03ee48e0e5044729de758da0c227c56, entries=150, sequenceid=308, filesize=12.0 K 2024-11-09T10:15:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/3cc56ace0f80431cac4a38809f6a55cc as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3cc56ace0f80431cac4a38809f6a55cc 2024-11-09T10:15:48,471 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3cc56ace0f80431cac4a38809f6a55cc, entries=150, sequenceid=308, filesize=12.0 K 2024-11-09T10:15:48,472 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~124.84 KB/127835, heapSize ~354.34 KB/362840, currentSize=53.67 KB/54960 for cac2483be99c556ef585845f10246658 in 946ms, sequenceid=308, compaction requested=false 2024-11-09T10:15:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-09T10:15:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-09T10:15:48,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T10:15:48,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1860 sec 2024-11-09T10:15:48,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1990 sec 2024-11-09T10:15:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:48,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T10:15:48,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:48,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:48,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:48,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:48,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:48,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:48,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/a06fc6c1a5e046aabc9c149e00181c4e is 50, key is test_row_0/A:col10/1731147348333/Put/seqid=0 2024-11-09T10:15:48,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741897_1073 (size=14741) 2024-11-09T10:15:48,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147408762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:48,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147408869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:49,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147409079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:49,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/a06fc6c1a5e046aabc9c149e00181c4e 2024-11-09T10:15:49,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/5200bc7c62104014b64d2a8cdaf8dd28 is 50, key is test_row_0/B:col10/1731147348333/Put/seqid=0 2024-11-09T10:15:49,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741898_1074 (size=12301) 2024-11-09T10:15:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147409389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:49,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/5200bc7c62104014b64d2a8cdaf8dd28 2024-11-09T10:15:49,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/37fd9fc3af1148feb8a4e43104d19097 is 50, key is test_row_0/C:col10/1731147348333/Put/seqid=0 2024-11-09T10:15:49,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741899_1075 (size=12301) 2024-11-09T10:15:49,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147409899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:49,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/37fd9fc3af1148feb8a4e43104d19097 2024-11-09T10:15:50,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/a06fc6c1a5e046aabc9c149e00181c4e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/a06fc6c1a5e046aabc9c149e00181c4e 2024-11-09T10:15:50,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/a06fc6c1a5e046aabc9c149e00181c4e, entries=200, sequenceid=323, filesize=14.4 K 2024-11-09T10:15:50,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/5200bc7c62104014b64d2a8cdaf8dd28 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5200bc7c62104014b64d2a8cdaf8dd28 2024-11-09T10:15:50,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5200bc7c62104014b64d2a8cdaf8dd28, entries=150, sequenceid=323, filesize=12.0 K 2024-11-09T10:15:50,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/37fd9fc3af1148feb8a4e43104d19097 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/37fd9fc3af1148feb8a4e43104d19097 2024-11-09T10:15:50,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/37fd9fc3af1148feb8a4e43104d19097, entries=150, sequenceid=323, filesize=12.0 K 2024-11-09T10:15:50,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cac2483be99c556ef585845f10246658 in 1377ms, sequenceid=323, compaction requested=true 2024-11-09T10:15:50,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:50,041 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:50,041 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:50,042 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:50,042 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:50,042 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:50,042 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:50,042 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,042 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,042 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/80ebb97782004dfca0beb11d53934b86, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e03ee48e0e5044729de758da0c227c56, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5200bc7c62104014b64d2a8cdaf8dd28] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=36.6 K 2024-11-09T10:15:50,042 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e0014759961c4482ad5533698fcff9d9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6103be6e27394b9daf75a7413c0d0ea9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/a06fc6c1a5e046aabc9c149e00181c4e] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=39.0 K 2024-11-09T10:15:50,043 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ebb97782004dfca0beb11d53934b86, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731147343915 2024-11-09T10:15:50,043 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0014759961c4482ad5533698fcff9d9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731147343915 2024-11-09T10:15:50,043 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e03ee48e0e5044729de758da0c227c56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731147346203 2024-11-09T10:15:50,043 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6103be6e27394b9daf75a7413c0d0ea9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731147346203 2024-11-09T10:15:50,043 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5200bc7c62104014b64d2a8cdaf8dd28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731147348329 2024-11-09T10:15:50,044 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a06fc6c1a5e046aabc9c149e00181c4e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731147348329 2024-11-09T10:15:50,052 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:50,052 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:50,053 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fbb494b1d5884a66a8ca0ecb52654cd5 is 50, key is test_row_0/B:col10/1731147348333/Put/seqid=0 2024-11-09T10:15:50,053 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/6dae0c85442640c5909634c743a80115 is 50, key is test_row_0/A:col10/1731147348333/Put/seqid=0 2024-11-09T10:15:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741900_1076 (size=13017) 2024-11-09T10:15:50,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741901_1077 (size=13017) 2024-11-09T10:15:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:50,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T10:15:50,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:50,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:50,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/1d488241d5834a8ea76d9a82aece1723 is 50, key is test_row_0/A:col10/1731147350412/Put/seqid=0 2024-11-09T10:15:50,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741902_1078 (size=17181) 2024-11-09T10:15:50,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147410436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147410437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147410438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58042 deadline: 1731147410439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,446 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T10:15:50,461 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:50,462 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-09T10:15:50,473 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/6dae0c85442640c5909634c743a80115 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6dae0c85442640c5909634c743a80115 2024-11-09T10:15:50,474 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T10:15:50,475 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:50,475 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/fbb494b1d5884a66a8ca0ecb52654cd5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fbb494b1d5884a66a8ca0ecb52654cd5 2024-11-09T10:15:50,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:50,483 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into 6dae0c85442640c5909634c743a80115(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:50,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:50,484 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=13, startTime=1731147350041; duration=0sec 2024-11-09T10:15:50,484 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:50,484 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:50,484 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:50,485 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:50,485 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/C is initiating minor compaction (all files) 2024-11-09T10:15:50,485 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/C in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,486 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into fbb494b1d5884a66a8ca0ecb52654cd5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:50,486 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/ea3cfa5dc7694248a4f53bea47a974c5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3cc56ace0f80431cac4a38809f6a55cc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/37fd9fc3af1148feb8a4e43104d19097] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=36.6 K 2024-11-09T10:15:50,486 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:50,486 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=13, startTime=1731147350041; duration=0sec 2024-11-09T10:15:50,486 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:50,486 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:50,486 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea3cfa5dc7694248a4f53bea47a974c5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731147343915 2024-11-09T10:15:50,487 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc56ace0f80431cac4a38809f6a55cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731147346203 2024-11-09T10:15:50,487 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37fd9fc3af1148feb8a4e43104d19097, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731147348329 2024-11-09T10:15:50,496 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:50,497 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/7b837c122aa84aac9509dabad41d8e67 is 50, key is test_row_0/C:col10/1731147348333/Put/seqid=0 2024-11-09T10:15:50,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741903_1079 (size=13017) 2024-11-09T10:15:50,516 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/7b837c122aa84aac9509dabad41d8e67 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/7b837c122aa84aac9509dabad41d8e67 2024-11-09T10:15:50,523 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/C of cac2483be99c556ef585845f10246658 into 7b837c122aa84aac9509dabad41d8e67(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:50,523 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:50,523 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/C, priority=13, startTime=1731147350041; duration=0sec 2024-11-09T10:15:50,523 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:50,523 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:50,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147410548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147410548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147410549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T10:15:50,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T10:15:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147410759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147410769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147410770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T10:15:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T10:15:50,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/1d488241d5834a8ea76d9a82aece1723 2024-11-09T10:15:50,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/ddea0a460490427888b3b236c29584bc is 50, key is test_row_0/B:col10/1731147350412/Put/seqid=0 2024-11-09T10:15:50,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741904_1080 (size=12301) 2024-11-09T10:15:50,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/ddea0a460490427888b3b236c29584bc 2024-11-09T10:15:50,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/031590af80204de0b88708cd6624c748 is 50, key is test_row_0/C:col10/1731147350412/Put/seqid=0 2024-11-09T10:15:50,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741905_1081 (size=12301) 2024-11-09T10:15:50,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147410919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:50,924 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:50,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T10:15:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:50,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:51,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147411079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147411079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147411079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T10:15:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:51,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T10:15:51,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T10:15:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:51,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:51,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/031590af80204de0b88708cd6624c748 2024-11-09T10:15:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/1d488241d5834a8ea76d9a82aece1723 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/1d488241d5834a8ea76d9a82aece1723 2024-11-09T10:15:51,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/1d488241d5834a8ea76d9a82aece1723, entries=250, sequenceid=345, filesize=16.8 K 2024-11-09T10:15:51,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/ddea0a460490427888b3b236c29584bc as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ddea0a460490427888b3b236c29584bc 2024-11-09T10:15:51,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ddea0a460490427888b3b236c29584bc, entries=150, sequenceid=345, filesize=12.0 K 2024-11-09T10:15:51,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/031590af80204de0b88708cd6624c748 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/031590af80204de0b88708cd6624c748 2024-11-09T10:15:51,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/031590af80204de0b88708cd6624c748, entries=150, sequenceid=345, filesize=12.0 K 2024-11-09T10:15:51,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for cac2483be99c556ef585845f10246658 in 922ms, sequenceid=345, compaction requested=false 2024-11-09T10:15:51,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:51,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T10:15:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:51,399 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:15:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:51,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/c903e41e942f46d8b84f39b606d03a76 is 50, key is test_row_0/A:col10/1731147350435/Put/seqid=0 2024-11-09T10:15:51,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741906_1082 (size=12301) 2024-11-09T10:15:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T10:15:51,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147351612, max timestamp=1731147351626Num uniques 859; 2024-11-09T10:15:51,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:15:51,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147351612, max timestamp=1731147351626Num uniques 859; 2024-11-09T10:15:51,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:15:51,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147351612, max timestamp=1731147351626Num uniques 859; 2024-11-09T10:15:51,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:15:51,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147411669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147411671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147411673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147411779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147411779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147411788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:51,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:51,810 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/c903e41e942f46d8b84f39b606d03a76 2024-11-09T10:15:51,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/c82b367266664c25911136b952c71c09 is 50, key is test_row_0/B:col10/1731147350435/Put/seqid=0 2024-11-09T10:15:51,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741907_1083 (size=12301) 2024-11-09T10:15:51,828 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/c82b367266664c25911136b952c71c09 2024-11-09T10:15:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/21144fb68b7a41758cf62ea10dd55280 is 50, key is test_row_0/C:col10/1731147350435/Put/seqid=0 2024-11-09T10:15:51,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741908_1084 (size=12301) 2024-11-09T10:15:51,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/21144fb68b7a41758cf62ea10dd55280 2024-11-09T10:15:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/c903e41e942f46d8b84f39b606d03a76 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c903e41e942f46d8b84f39b606d03a76 2024-11-09T10:15:51,865 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c903e41e942f46d8b84f39b606d03a76, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T10:15:51,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/c82b367266664c25911136b952c71c09 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/c82b367266664c25911136b952c71c09 2024-11-09T10:15:51,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/c82b367266664c25911136b952c71c09, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T10:15:51,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/21144fb68b7a41758cf62ea10dd55280 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21144fb68b7a41758cf62ea10dd55280 2024-11-09T10:15:51,883 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21144fb68b7a41758cf62ea10dd55280, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T10:15:51,884 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cac2483be99c556ef585845f10246658 in 485ms, sequenceid=358, compaction requested=true 2024-11-09T10:15:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-09T10:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-09T10:15:51,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T10:15:51,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-09T10:15:51,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4260 sec 2024-11-09T10:15:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:51,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-09T10:15:51,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:51,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:51,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:51,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:51,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:51,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:15:51,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/7f2a6dce5d8a4b0ca346da6ff4e59a48 is 50, key is test_row_0/A:col10/1731147351668/Put/seqid=0 2024-11-09T10:15:52,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741909_1085 (size=14741) 2024-11-09T10:15:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147412001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147412002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147412003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147412119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147412119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147412119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147412329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147412329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147412329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/7f2a6dce5d8a4b0ca346da6ff4e59a48 2024-11-09T10:15:52,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bb8891f15eaa41698586d231bbac0e69 is 50, key is test_row_0/B:col10/1731147351668/Put/seqid=0 2024-11-09T10:15:52,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741910_1086 (size=12301) 2024-11-09T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T10:15:52,629 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:52,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-09T10:15:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T10:15:52,631 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:15:52,632 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:15:52,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:15:52,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147412639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147412640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147412640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T10:15:52,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:52,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:52,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:52,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:52,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bb8891f15eaa41698586d231bbac0e69 2024-11-09T10:15:52,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4abd705050de485fa886424f1a3d9d07 is 50, key is test_row_0/C:col10/1731147351668/Put/seqid=0 2024-11-09T10:15:52,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741911_1087 (size=12301) 2024-11-09T10:15:52,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4abd705050de485fa886424f1a3d9d07 2024-11-09T10:15:52,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/7f2a6dce5d8a4b0ca346da6ff4e59a48 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/7f2a6dce5d8a4b0ca346da6ff4e59a48 2024-11-09T10:15:52,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/7f2a6dce5d8a4b0ca346da6ff4e59a48, entries=200, sequenceid=395, filesize=14.4 K 2024-11-09T10:15:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/bb8891f15eaa41698586d231bbac0e69 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bb8891f15eaa41698586d231bbac0e69 2024-11-09T10:15:52,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bb8891f15eaa41698586d231bbac0e69, entries=150, sequenceid=395, filesize=12.0 K 2024-11-09T10:15:52,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4abd705050de485fa886424f1a3d9d07 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4abd705050de485fa886424f1a3d9d07 2024-11-09T10:15:52,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4abd705050de485fa886424f1a3d9d07, entries=150, sequenceid=395, filesize=12.0 K 2024-11-09T10:15:52,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for cac2483be99c556ef585845f10246658 in 875ms, sequenceid=395, compaction requested=true 2024-11-09T10:15:52,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:52,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:52,867 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:52,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:52,867 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:15:52,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:52,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:52,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:52,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:52,871 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:52,871 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:15:52,871 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:52,871 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:52,871 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:52,871 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:52,872 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6dae0c85442640c5909634c743a80115, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/1d488241d5834a8ea76d9a82aece1723, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c903e41e942f46d8b84f39b606d03a76, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/7f2a6dce5d8a4b0ca346da6ff4e59a48] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=55.9 K 2024-11-09T10:15:52,872 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fbb494b1d5884a66a8ca0ecb52654cd5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ddea0a460490427888b3b236c29584bc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/c82b367266664c25911136b952c71c09, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bb8891f15eaa41698586d231bbac0e69] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=48.8 K 2024-11-09T10:15:52,872 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dae0c85442640c5909634c743a80115, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731147348329 2024-11-09T10:15:52,872 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb494b1d5884a66a8ca0ecb52654cd5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731147348329 2024-11-09T10:15:52,873 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d488241d5834a8ea76d9a82aece1723, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731147348733 2024-11-09T10:15:52,873 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting ddea0a460490427888b3b236c29584bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731147348759 2024-11-09T10:15:52,873 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c903e41e942f46d8b84f39b606d03a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731147350425 2024-11-09T10:15:52,873 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c82b367266664c25911136b952c71c09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731147350425 2024-11-09T10:15:52,874 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2a6dce5d8a4b0ca346da6ff4e59a48, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731147351654 2024-11-09T10:15:52,874 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8891f15eaa41698586d231bbac0e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731147351654 2024-11-09T10:15:52,888 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:52,888 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/792e61511e854467a1517942058eb8f1 is 50, key is test_row_0/A:col10/1731147351668/Put/seqid=0 2024-11-09T10:15:52,891 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:52,892 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/4550025deef54de29deaa64858e486ef is 50, key is test_row_0/B:col10/1731147351668/Put/seqid=0 2024-11-09T10:15:52,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741912_1088 (size=13153) 2024-11-09T10:15:52,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741913_1089 (size=13153) 2024-11-09T10:15:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:52,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:15:52,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:52,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:52,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:52,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:52,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:52,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:52,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/5a0be07b2088417ca6a418bb0c98d356 is 50, key is test_row_0/A:col10/1731147352928/Put/seqid=0 2024-11-09T10:15:52,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:52,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:52,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:52,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:52,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:52,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:52,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741914_1090 (size=14741) 2024-11-09T10:15:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T10:15:52,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/5a0be07b2088417ca6a418bb0c98d356 2024-11-09T10:15:52,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/57a7f3de2960490693768fca32237987 is 50, key is test_row_0/B:col10/1731147352928/Put/seqid=0 2024-11-09T10:15:52,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741915_1091 (size=12301) 2024-11-09T10:15:52,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/57a7f3de2960490693768fca32237987 2024-11-09T10:15:52,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2c49da7c373d452c97cce6171393b5ea is 50, key is test_row_0/C:col10/1731147352928/Put/seqid=0 2024-11-09T10:15:52,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741916_1092 (size=12301) 2024-11-09T10:15:52,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2c49da7c373d452c97cce6171393b5ea 2024-11-09T10:15:52,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/5a0be07b2088417ca6a418bb0c98d356 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/5a0be07b2088417ca6a418bb0c98d356 2024-11-09T10:15:53,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/5a0be07b2088417ca6a418bb0c98d356, entries=200, sequenceid=405, filesize=14.4 K 2024-11-09T10:15:53,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/57a7f3de2960490693768fca32237987 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/57a7f3de2960490693768fca32237987 2024-11-09T10:15:53,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/57a7f3de2960490693768fca32237987, entries=150, sequenceid=405, filesize=12.0 K 2024-11-09T10:15:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/2c49da7c373d452c97cce6171393b5ea as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2c49da7c373d452c97cce6171393b5ea 2024-11-09T10:15:53,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2c49da7c373d452c97cce6171393b5ea, entries=150, sequenceid=405, filesize=12.0 K 2024-11-09T10:15:53,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=87.22 KB/89310 for cac2483be99c556ef585845f10246658 in 86ms, sequenceid=405, compaction requested=true 2024-11-09T10:15:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:15:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T10:15:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-09T10:15:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:53,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T10:15:53,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:53,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:53,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:53,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:53,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:53,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:53,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/af6ff73dcc3149059d77a0ff8769d9c2 is 50, key is test_row_0/A:col10/1731147353003/Put/seqid=0 2024-11-09T10:15:53,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741917_1093 (size=17181) 2024-11-09T10:15:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147413077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:53,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147413149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147413149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147413149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147413188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:53,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:53,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T10:15:53,316 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/792e61511e854467a1517942058eb8f1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/792e61511e854467a1517942058eb8f1 2024-11-09T10:15:53,325 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into 792e61511e854467a1517942058eb8f1(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:53,325 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:53,325 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=12, startTime=1731147352867; duration=0sec 2024-11-09T10:15:53,325 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/4550025deef54de29deaa64858e486ef as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/4550025deef54de29deaa64858e486ef 2024-11-09T10:15:53,325 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-09T10:15:53,325 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:53,325 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:53,325 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T10:15:53,328 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T10:15:53,328 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/C is initiating minor compaction (all files) 2024-11-09T10:15:53,328 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/C in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,328 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/7b837c122aa84aac9509dabad41d8e67, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/031590af80204de0b88708cd6624c748, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21144fb68b7a41758cf62ea10dd55280, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4abd705050de485fa886424f1a3d9d07, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2c49da7c373d452c97cce6171393b5ea] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=60.8 K 2024-11-09T10:15:53,329 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b837c122aa84aac9509dabad41d8e67, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731147348329 2024-11-09T10:15:53,330 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 031590af80204de0b88708cd6624c748, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731147348759 2024-11-09T10:15:53,332 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21144fb68b7a41758cf62ea10dd55280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731147350425 2024-11-09T10:15:53,333 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into 4550025deef54de29deaa64858e486ef(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:53,333 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:53,333 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4abd705050de485fa886424f1a3d9d07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731147351654 2024-11-09T10:15:53,333 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=12, startTime=1731147352867; duration=0sec 2024-11-09T10:15:53,333 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c49da7c373d452c97cce6171393b5ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731147351995 2024-11-09T10:15:53,334 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T10:15:53,334 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:53,334 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:53,334 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-09T10:15:53,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T10:15:53,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T10:15:53,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. because compaction request was cancelled 2024-11-09T10:15:53,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:53,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T10:15:53,337 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T10:15:53,337 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T10:15:53,337 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. because compaction request was cancelled 2024-11-09T10:15:53,337 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:53,337 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T10:15:53,338 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T10:15:53,338 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T10:15:53,338 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. because compaction request was cancelled 2024-11-09T10:15:53,338 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:53,349 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#C#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:53,350 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4ffd2b0b8d124fd7a2c901e820c67e70 is 50, key is test_row_0/C:col10/1731147352928/Put/seqid=0 2024-11-09T10:15:53,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741918_1094 (size=13187) 2024-11-09T10:15:53,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:53,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147413399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/af6ff73dcc3149059d77a0ff8769d9c2 2024-11-09T10:15:53,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8534c2b7188d48b6850b73fb40434d47 is 50, key is test_row_0/B:col10/1731147353003/Put/seqid=0 2024-11-09T10:15:53,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741919_1095 (size=12301) 2024-11-09T10:15:53,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147413709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:53,768 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/4ffd2b0b8d124fd7a2c901e820c67e70 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4ffd2b0b8d124fd7a2c901e820c67e70 2024-11-09T10:15:53,776 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cac2483be99c556ef585845f10246658/C of cac2483be99c556ef585845f10246658 into 4ffd2b0b8d124fd7a2c901e820c67e70(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:53,776 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:53,776 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/C, priority=11, startTime=1731147353017; duration=0sec 2024-11-09T10:15:53,776 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:53,776 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T10:15:53,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8534c2b7188d48b6850b73fb40434d47 2024-11-09T10:15:53,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:53,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:53,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/30a1634e2fa9472880bd85870463d111 is 50, key is test_row_0/C:col10/1731147353003/Put/seqid=0 2024-11-09T10:15:53,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741920_1096 (size=12301) 2024-11-09T10:15:54,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:54,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:54,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58016 deadline: 1731147414159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:54,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:54,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58102 deadline: 1731147414159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1731147414159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:54,164 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:54,164 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:54,164 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in cac2483be99c556ef585845f10246658 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:15:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:54,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:54,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:15:54,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:15:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58030 deadline: 1731147414219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:15:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:15:54,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/30a1634e2fa9472880bd85870463d111 2024-11-09T10:15:54,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/af6ff73dcc3149059d77a0ff8769d9c2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/af6ff73dcc3149059d77a0ff8769d9c2 2024-11-09T10:15:54,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/af6ff73dcc3149059d77a0ff8769d9c2, entries=250, sequenceid=422, filesize=16.8 K 2024-11-09T10:15:54,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8534c2b7188d48b6850b73fb40434d47 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8534c2b7188d48b6850b73fb40434d47 2024-11-09T10:15:54,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8534c2b7188d48b6850b73fb40434d47, entries=150, sequenceid=422, filesize=12.0 K 2024-11-09T10:15:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/30a1634e2fa9472880bd85870463d111 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/30a1634e2fa9472880bd85870463d111 2024-11-09T10:15:54,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/30a1634e2fa9472880bd85870463d111, entries=150, sequenceid=422, filesize=12.0 K 2024-11-09T10:15:54,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cac2483be99c556ef585845f10246658 in 1278ms, sequenceid=422, compaction requested=true 2024-11-09T10:15:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:15:54,299 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:54,299 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:15:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:15:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cac2483be99c556ef585845f10246658:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:15:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:54,300 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:54,300 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:15:54,300 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/B is initiating minor compaction (all files) 2024-11-09T10:15:54,300 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): cac2483be99c556ef585845f10246658/A is initiating minor compaction (all files) 2024-11-09T10:15:54,300 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/B in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:54,300 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cac2483be99c556ef585845f10246658/A in TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:54,300 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/4550025deef54de29deaa64858e486ef, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/57a7f3de2960490693768fca32237987, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8534c2b7188d48b6850b73fb40434d47] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=36.9 K 2024-11-09T10:15:54,300 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/792e61511e854467a1517942058eb8f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/5a0be07b2088417ca6a418bb0c98d356, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/af6ff73dcc3149059d77a0ff8769d9c2] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp, totalSize=44.0 K 2024-11-09T10:15:54,301 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4550025deef54de29deaa64858e486ef, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731147351654 2024-11-09T10:15:54,301 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792e61511e854467a1517942058eb8f1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731147351654 2024-11-09T10:15:54,301 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a7f3de2960490693768fca32237987, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731147351995 2024-11-09T10:15:54,301 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0be07b2088417ca6a418bb0c98d356, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731147351995 2024-11-09T10:15:54,301 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 8534c2b7188d48b6850b73fb40434d47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731147353003 2024-11-09T10:15:54,301 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting af6ff73dcc3149059d77a0ff8769d9c2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731147352991 2024-11-09T10:15:54,308 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:54,308 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cac2483be99c556ef585845f10246658#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:15:54,308 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/ef812304070d4d72ac9981aae2436a8e is 50, key is test_row_0/A:col10/1731147353003/Put/seqid=0 2024-11-09T10:15:54,309 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8bc97c100e0d4780992743a0a37862f3 is 50, key is test_row_0/B:col10/1731147353003/Put/seqid=0 2024-11-09T10:15:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741921_1097 (size=13255) 2024-11-09T10:15:54,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741922_1098 (size=13255) 2024-11-09T10:15:54,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T10:15:54,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:54,319 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:15:54,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:54,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/76794a1475d546c6b8817df66ba13a99 is 50, key is test_row_0/A:col10/1731147353062/Put/seqid=0 2024-11-09T10:15:54,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741923_1099 (size=12301) 2024-11-09T10:15:54,340 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/76794a1475d546c6b8817df66ba13a99 2024-11-09T10:15:54,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8673838672bf45168a113d5bd8ad06af is 50, key is test_row_0/B:col10/1731147353062/Put/seqid=0 2024-11-09T10:15:54,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741924_1100 (size=12301) 2024-11-09T10:15:54,367 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-09T10:15:54,367 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-09T10:15:54,367 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-09T10:15:54,367 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:54,367 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:54,367 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:54,372 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,372 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,372 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,373 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,373 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:54,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:54,373 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:54,374 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-09T10:15:54,374 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:54,374 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,375 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on cac2483be99c556ef585845f10246658 2024-11-09T10:15:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. as already flushing 2024-11-09T10:15:54,481 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-09T10:15:54,482 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:54,482 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,482 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:54,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:54,724 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8bc97c100e0d4780992743a0a37862f3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8bc97c100e0d4780992743a0a37862f3 2024-11-09T10:15:54,727 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/ef812304070d4d72ac9981aae2436a8e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/ef812304070d4d72ac9981aae2436a8e 2024-11-09T10:15:54,732 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/B of cac2483be99c556ef585845f10246658 into 8bc97c100e0d4780992743a0a37862f3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:54,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:54,732 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/B, priority=13, startTime=1731147354299; duration=0sec 2024-11-09T10:15:54,732 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cac2483be99c556ef585845f10246658/A of cac2483be99c556ef585845f10246658 into ef812304070d4d72ac9981aae2436a8e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:15:54,732 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:54,732 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., storeName=cac2483be99c556ef585845f10246658/A, priority=13, startTime=1731147354299; duration=0sec 2024-11-09T10:15:54,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:15:54,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:B 2024-11-09T10:15:54,732 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:15:54,732 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:A 2024-11-09T10:15:54,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T10:15:54,733 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T10:15:54,733 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T10:15:54,733 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. because compaction request was cancelled 2024-11-09T10:15:54,733 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cac2483be99c556ef585845f10246658:C 2024-11-09T10:15:54,756 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8673838672bf45168a113d5bd8ad06af 2024-11-09T10:15:54,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/69abe007421b4919954b042bf3323f22 is 50, key is test_row_0/C:col10/1731147353062/Put/seqid=0 2024-11-09T10:15:54,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741925_1101 (size=12301) 2024-11-09T10:15:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T10:15:55,173 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/69abe007421b4919954b042bf3323f22 2024-11-09T10:15:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/76794a1475d546c6b8817df66ba13a99 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/76794a1475d546c6b8817df66ba13a99 2024-11-09T10:15:55,187 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/76794a1475d546c6b8817df66ba13a99, entries=150, sequenceid=440, filesize=12.0 K 2024-11-09T10:15:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/8673838672bf45168a113d5bd8ad06af as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8673838672bf45168a113d5bd8ad06af 2024-11-09T10:15:55,193 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8673838672bf45168a113d5bd8ad06af, entries=150, sequenceid=440, filesize=12.0 K 2024-11-09T10:15:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/69abe007421b4919954b042bf3323f22 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/69abe007421b4919954b042bf3323f22 2024-11-09T10:15:55,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/69abe007421b4919954b042bf3323f22, entries=150, sequenceid=440, filesize=12.0 K 2024-11-09T10:15:55,201 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for cac2483be99c556ef585845f10246658 in 882ms, sequenceid=440, compaction requested=true 2024-11-09T10:15:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for cac2483be99c556ef585845f10246658: 2024-11-09T10:15:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-09T10:15:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-09T10:15:55,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T10:15:55,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5710 sec 2024-11-09T10:15:55,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.5770 sec 2024-11-09T10:15:55,230 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-09T10:15:55,230 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:55,230 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:55,230 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:55,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:56,173 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-09T10:15:56,173 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:56,173 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,173 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:56,181 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-09T10:15:56,181 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-09T10:15:56,181 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:56,181 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:15:56,181 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,181 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,181 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,181 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:56,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T10:15:56,819 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:56,819 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T10:15:56,819 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2704 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2807 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1229 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3682 rows 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1223 2024-11-09T10:15:56,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3664 rows 2024-11-09T10:15:56,820 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:15:56,821 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:15:56,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:15:56,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:15:56,826 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T10:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T10:15:56,834 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147356834"}]},"ts":"1731147356834"} 2024-11-09T10:15:56,836 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T10:15:56,837 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T10:15:56,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:15:56,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, UNASSIGN}] 2024-11-09T10:15:56,845 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, UNASSIGN 2024-11-09T10:15:56,847 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cac2483be99c556ef585845f10246658, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:15:56,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, UNASSIGN because future has completed 2024-11-09T10:15:56,850 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:15:56,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:15:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T10:15:56,956 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T10:15:57,008 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close cac2483be99c556ef585845f10246658 2024-11-09T10:15:57,008 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:15:57,009 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing cac2483be99c556ef585845f10246658, disabling compactions & flushes 2024-11-09T10:15:57,009 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:57,009 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:57,009 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. after waiting 0 ms 2024-11-09T10:15:57,009 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:57,010 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing cac2483be99c556ef585845f10246658 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T10:15:57,010 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=A 2024-11-09T10:15:57,010 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:57,010 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=B 2024-11-09T10:15:57,010 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:57,010 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cac2483be99c556ef585845f10246658, store=C 2024-11-09T10:15:57,010 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:15:57,017 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/df7c7933105641178cbe9fed41f0811f is 50, key is test_row_0/A:col10/1731147356170/Put/seqid=0 2024-11-09T10:15:57,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741926_1102 (size=12301) 2024-11-09T10:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T10:15:57,422 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/df7c7933105641178cbe9fed41f0811f 2024-11-09T10:15:57,434 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/59c73980ff7244bc80e8f927a4dc6c28 is 50, key is test_row_0/B:col10/1731147356170/Put/seqid=0 2024-11-09T10:15:57,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741927_1103 (size=12301) 2024-11-09T10:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T10:15:57,838 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/59c73980ff7244bc80e8f927a4dc6c28 2024-11-09T10:15:57,845 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/be168a9bf2d74fe88400bb187ef187a7 is 50, key is test_row_0/C:col10/1731147356170/Put/seqid=0 2024-11-09T10:15:57,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741928_1104 (size=12301) 2024-11-09T10:15:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T10:15:58,249 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/be168a9bf2d74fe88400bb187ef187a7 2024-11-09T10:15:58,253 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/A/df7c7933105641178cbe9fed41f0811f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/df7c7933105641178cbe9fed41f0811f 2024-11-09T10:15:58,258 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/df7c7933105641178cbe9fed41f0811f, entries=150, sequenceid=450, filesize=12.0 K 2024-11-09T10:15:58,259 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/B/59c73980ff7244bc80e8f927a4dc6c28 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/59c73980ff7244bc80e8f927a4dc6c28 2024-11-09T10:15:58,265 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/59c73980ff7244bc80e8f927a4dc6c28, entries=150, sequenceid=450, filesize=12.0 K 2024-11-09T10:15:58,266 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/.tmp/C/be168a9bf2d74fe88400bb187ef187a7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/be168a9bf2d74fe88400bb187ef187a7 2024-11-09T10:15:58,270 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/be168a9bf2d74fe88400bb187ef187a7, entries=150, sequenceid=450, filesize=12.0 K 2024-11-09T10:15:58,271 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cac2483be99c556ef585845f10246658 in 1262ms, sequenceid=450, compaction requested=true 2024-11-09T10:15:58,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/afa85605a5d54f9da7e9a9dc2e52a4dd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b54d28bc7e9a42248485df1265ee268a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6b9026ecbf042939f4e14b924ace161, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/04cc358d99d64e878bb784b64dd3bb57, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/24f23a8790914f1cbbd488de8b9edfc8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/42e3620bc769459b9c9623534f4bd282, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/d3253f37afbd446f90c854e642061abf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/89c19d6b21144f31a1116624e11b0f4f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/9217a9e17e724983bc1037e90f7db46a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/30083a02179146eebe45f5657e9963b1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6652dd73acb471ab83d1a07c31ecb0b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/3b5f7dbcab494d76b0e52f674e298ea6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/68898c7aaf024c0a94ecc025914ba907, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b0cdae342e2d42c09061093d7736e239, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/55d1d230a1d84e2596e4b79ca7c3f258, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/2e485bfed72e4176b54bcd54d1abc0bb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b94dcf27b1ed45519c92821f8abccee6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e0014759961c4482ad5533698fcff9d9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c3a2d98d184643be8dd350ec15063f6e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6103be6e27394b9daf75a7413c0d0ea9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/a06fc6c1a5e046aabc9c149e00181c4e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6dae0c85442640c5909634c743a80115, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/1d488241d5834a8ea76d9a82aece1723, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c903e41e942f46d8b84f39b606d03a76, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/7f2a6dce5d8a4b0ca346da6ff4e59a48, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/792e61511e854467a1517942058eb8f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/5a0be07b2088417ca6a418bb0c98d356, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/af6ff73dcc3149059d77a0ff8769d9c2] to archive 2024-11-09T10:15:58,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:15:58,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/afa85605a5d54f9da7e9a9dc2e52a4dd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/afa85605a5d54f9da7e9a9dc2e52a4dd 2024-11-09T10:15:58,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b54d28bc7e9a42248485df1265ee268a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b54d28bc7e9a42248485df1265ee268a 2024-11-09T10:15:58,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6b9026ecbf042939f4e14b924ace161 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6b9026ecbf042939f4e14b924ace161 2024-11-09T10:15:58,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/04cc358d99d64e878bb784b64dd3bb57 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/04cc358d99d64e878bb784b64dd3bb57 2024-11-09T10:15:58,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/24f23a8790914f1cbbd488de8b9edfc8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/24f23a8790914f1cbbd488de8b9edfc8 2024-11-09T10:15:58,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/42e3620bc769459b9c9623534f4bd282 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/42e3620bc769459b9c9623534f4bd282 2024-11-09T10:15:58,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/d3253f37afbd446f90c854e642061abf to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/d3253f37afbd446f90c854e642061abf 2024-11-09T10:15:58,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/89c19d6b21144f31a1116624e11b0f4f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/89c19d6b21144f31a1116624e11b0f4f 2024-11-09T10:15:58,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/9217a9e17e724983bc1037e90f7db46a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/9217a9e17e724983bc1037e90f7db46a 2024-11-09T10:15:58,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/30083a02179146eebe45f5657e9963b1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/30083a02179146eebe45f5657e9963b1 2024-11-09T10:15:58,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6652dd73acb471ab83d1a07c31ecb0b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e6652dd73acb471ab83d1a07c31ecb0b 2024-11-09T10:15:58,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/3b5f7dbcab494d76b0e52f674e298ea6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/3b5f7dbcab494d76b0e52f674e298ea6 2024-11-09T10:15:58,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/68898c7aaf024c0a94ecc025914ba907 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/68898c7aaf024c0a94ecc025914ba907 2024-11-09T10:15:58,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b0cdae342e2d42c09061093d7736e239 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b0cdae342e2d42c09061093d7736e239 2024-11-09T10:15:58,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/55d1d230a1d84e2596e4b79ca7c3f258 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/55d1d230a1d84e2596e4b79ca7c3f258 2024-11-09T10:15:58,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/2e485bfed72e4176b54bcd54d1abc0bb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/2e485bfed72e4176b54bcd54d1abc0bb 2024-11-09T10:15:58,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b94dcf27b1ed45519c92821f8abccee6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/b94dcf27b1ed45519c92821f8abccee6 2024-11-09T10:15:58,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e0014759961c4482ad5533698fcff9d9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/e0014759961c4482ad5533698fcff9d9 2024-11-09T10:15:58,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c3a2d98d184643be8dd350ec15063f6e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c3a2d98d184643be8dd350ec15063f6e 2024-11-09T10:15:58,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6103be6e27394b9daf75a7413c0d0ea9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6103be6e27394b9daf75a7413c0d0ea9 2024-11-09T10:15:58,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/a06fc6c1a5e046aabc9c149e00181c4e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/a06fc6c1a5e046aabc9c149e00181c4e 2024-11-09T10:15:58,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6dae0c85442640c5909634c743a80115 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/6dae0c85442640c5909634c743a80115 2024-11-09T10:15:58,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/1d488241d5834a8ea76d9a82aece1723 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/1d488241d5834a8ea76d9a82aece1723 2024-11-09T10:15:58,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c903e41e942f46d8b84f39b606d03a76 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/c903e41e942f46d8b84f39b606d03a76 2024-11-09T10:15:58,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/7f2a6dce5d8a4b0ca346da6ff4e59a48 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/7f2a6dce5d8a4b0ca346da6ff4e59a48 2024-11-09T10:15:58,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/792e61511e854467a1517942058eb8f1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/792e61511e854467a1517942058eb8f1 2024-11-09T10:15:58,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/5a0be07b2088417ca6a418bb0c98d356 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/5a0be07b2088417ca6a418bb0c98d356 2024-11-09T10:15:58,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/af6ff73dcc3149059d77a0ff8769d9c2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/af6ff73dcc3149059d77a0ff8769d9c2 2024-11-09T10:15:58,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e720c0b80eba45648bd2a3d801a314b2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7843c7eaed2f4cd293d22a998d5f0921, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/41cd2f397b914cdcbf07109dd6458299, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7116e9d9ca8d4e8a8d1ebde3b5bd50dd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fea3d51582454e898ead0d291fc49f27, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/08f09161fc484b178a42a7792519f5e8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/2f1328129dea4842a754102e435a2f77, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/6c3b631e2c6e4221ab155a9fa743c5c5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/dd3ae459b6494d1e815fb5db1d30d243, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd87185b5ed5401ea9b071782dbb73d1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5eafdaa93e4d4f8e870acb3ea56fd4d7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd2d22b9c7d641e4abb84bf956c63c12, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ce13743aa6414c72b87c33ba4c9b9e76, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/961262bcb03c434c8e319ac7cfb2a35e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/edba07735dbf433b8beb99d802339499, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fe50d42495214442a59b3e7591335f17, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/3973e5d2c69143048e8861182934d78a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/80ebb97782004dfca0beb11d53934b86, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/1d3fc42f975d409a851b27bac975998f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e03ee48e0e5044729de758da0c227c56, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fbb494b1d5884a66a8ca0ecb52654cd5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5200bc7c62104014b64d2a8cdaf8dd28, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ddea0a460490427888b3b236c29584bc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/c82b367266664c25911136b952c71c09, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/4550025deef54de29deaa64858e486ef, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bb8891f15eaa41698586d231bbac0e69, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/57a7f3de2960490693768fca32237987, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8534c2b7188d48b6850b73fb40434d47] to archive 2024-11-09T10:15:58,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:15:58,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e720c0b80eba45648bd2a3d801a314b2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e720c0b80eba45648bd2a3d801a314b2 2024-11-09T10:15:58,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7843c7eaed2f4cd293d22a998d5f0921 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7843c7eaed2f4cd293d22a998d5f0921 2024-11-09T10:15:58,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/41cd2f397b914cdcbf07109dd6458299 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/41cd2f397b914cdcbf07109dd6458299 2024-11-09T10:15:58,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7116e9d9ca8d4e8a8d1ebde3b5bd50dd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/7116e9d9ca8d4e8a8d1ebde3b5bd50dd 2024-11-09T10:15:58,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fea3d51582454e898ead0d291fc49f27 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fea3d51582454e898ead0d291fc49f27 2024-11-09T10:15:58,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/08f09161fc484b178a42a7792519f5e8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/08f09161fc484b178a42a7792519f5e8 2024-11-09T10:15:58,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/2f1328129dea4842a754102e435a2f77 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/2f1328129dea4842a754102e435a2f77 2024-11-09T10:15:58,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/6c3b631e2c6e4221ab155a9fa743c5c5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/6c3b631e2c6e4221ab155a9fa743c5c5 2024-11-09T10:15:58,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/dd3ae459b6494d1e815fb5db1d30d243 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/dd3ae459b6494d1e815fb5db1d30d243 2024-11-09T10:15:58,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd87185b5ed5401ea9b071782dbb73d1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd87185b5ed5401ea9b071782dbb73d1 2024-11-09T10:15:58,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5eafdaa93e4d4f8e870acb3ea56fd4d7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5eafdaa93e4d4f8e870acb3ea56fd4d7 2024-11-09T10:15:58,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd2d22b9c7d641e4abb84bf956c63c12 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bd2d22b9c7d641e4abb84bf956c63c12 2024-11-09T10:15:58,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ce13743aa6414c72b87c33ba4c9b9e76 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ce13743aa6414c72b87c33ba4c9b9e76 2024-11-09T10:15:58,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/961262bcb03c434c8e319ac7cfb2a35e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/961262bcb03c434c8e319ac7cfb2a35e 2024-11-09T10:15:58,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/edba07735dbf433b8beb99d802339499 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/edba07735dbf433b8beb99d802339499 2024-11-09T10:15:58,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fe50d42495214442a59b3e7591335f17 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fe50d42495214442a59b3e7591335f17 2024-11-09T10:15:58,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/3973e5d2c69143048e8861182934d78a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/3973e5d2c69143048e8861182934d78a 2024-11-09T10:15:58,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/80ebb97782004dfca0beb11d53934b86 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/80ebb97782004dfca0beb11d53934b86 2024-11-09T10:15:58,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/1d3fc42f975d409a851b27bac975998f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/1d3fc42f975d409a851b27bac975998f 2024-11-09T10:15:58,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e03ee48e0e5044729de758da0c227c56 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/e03ee48e0e5044729de758da0c227c56 2024-11-09T10:15:58,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fbb494b1d5884a66a8ca0ecb52654cd5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/fbb494b1d5884a66a8ca0ecb52654cd5 2024-11-09T10:15:58,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5200bc7c62104014b64d2a8cdaf8dd28 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/5200bc7c62104014b64d2a8cdaf8dd28 2024-11-09T10:15:58,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ddea0a460490427888b3b236c29584bc to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/ddea0a460490427888b3b236c29584bc 2024-11-09T10:15:58,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/c82b367266664c25911136b952c71c09 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/c82b367266664c25911136b952c71c09 2024-11-09T10:15:58,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/4550025deef54de29deaa64858e486ef to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/4550025deef54de29deaa64858e486ef 2024-11-09T10:15:58,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bb8891f15eaa41698586d231bbac0e69 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/bb8891f15eaa41698586d231bbac0e69 2024-11-09T10:15:58,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/57a7f3de2960490693768fca32237987 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/57a7f3de2960490693768fca32237987 2024-11-09T10:15:58,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8534c2b7188d48b6850b73fb40434d47 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8534c2b7188d48b6850b73fb40434d47 2024-11-09T10:15:58,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2503f530f669445ab32be5438bb73adb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2b324bc37ada4624b022140725ae7843, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/138c260325e14d2e889c1fde2682e0a5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4932fc3ba5b6487a93d0fa8bdeac4a3c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4930b6a4b02e46a9bc939a22f52c6452, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/c3b2e30c2fa6486d845105954d7378af, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/dee9805bad22450ba2e9d6486ff44766, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/1e1cba6cf7464f1ca26d194b0a0d728a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/20f4205494f842a29851773e167438e1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/89c315d57e74439c8acf1755eecaf44e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4f3fcad3c62d45b689fc391d09f9306d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/5aab89a7402a4eaaa02a65e46e2007bc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4a5e8e9117954dbdb0da0486cb2beea2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3fb4da58b52b4b779b91eec8a31dd6ce, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/07281b71a0854bb8aa331162ff292bc5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/9dfa35bc3e7043628843d3cfffe9c2d8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/6354a920550f471aa8d7134d12c47941, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/ea3cfa5dc7694248a4f53bea47a974c5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21d8048de5004dcda82b0b9d453b8bfe, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3cc56ace0f80431cac4a38809f6a55cc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/7b837c122aa84aac9509dabad41d8e67, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/37fd9fc3af1148feb8a4e43104d19097, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/031590af80204de0b88708cd6624c748, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21144fb68b7a41758cf62ea10dd55280, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4abd705050de485fa886424f1a3d9d07, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2c49da7c373d452c97cce6171393b5ea] to archive 2024-11-09T10:15:58,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:15:58,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2503f530f669445ab32be5438bb73adb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2503f530f669445ab32be5438bb73adb 2024-11-09T10:15:58,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2b324bc37ada4624b022140725ae7843 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2b324bc37ada4624b022140725ae7843 2024-11-09T10:15:58,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/138c260325e14d2e889c1fde2682e0a5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/138c260325e14d2e889c1fde2682e0a5 2024-11-09T10:15:58,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4932fc3ba5b6487a93d0fa8bdeac4a3c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4932fc3ba5b6487a93d0fa8bdeac4a3c 2024-11-09T10:15:58,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4930b6a4b02e46a9bc939a22f52c6452 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4930b6a4b02e46a9bc939a22f52c6452 2024-11-09T10:15:58,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/c3b2e30c2fa6486d845105954d7378af to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/c3b2e30c2fa6486d845105954d7378af 2024-11-09T10:15:58,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/dee9805bad22450ba2e9d6486ff44766 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/dee9805bad22450ba2e9d6486ff44766 2024-11-09T10:15:58,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/1e1cba6cf7464f1ca26d194b0a0d728a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/1e1cba6cf7464f1ca26d194b0a0d728a 2024-11-09T10:15:58,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/20f4205494f842a29851773e167438e1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/20f4205494f842a29851773e167438e1 2024-11-09T10:15:58,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/89c315d57e74439c8acf1755eecaf44e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/89c315d57e74439c8acf1755eecaf44e 2024-11-09T10:15:58,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4f3fcad3c62d45b689fc391d09f9306d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4f3fcad3c62d45b689fc391d09f9306d 2024-11-09T10:15:58,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/5aab89a7402a4eaaa02a65e46e2007bc to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/5aab89a7402a4eaaa02a65e46e2007bc 2024-11-09T10:15:58,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4a5e8e9117954dbdb0da0486cb2beea2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4a5e8e9117954dbdb0da0486cb2beea2 2024-11-09T10:15:58,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3fb4da58b52b4b779b91eec8a31dd6ce to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3fb4da58b52b4b779b91eec8a31dd6ce 2024-11-09T10:15:58,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/07281b71a0854bb8aa331162ff292bc5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/07281b71a0854bb8aa331162ff292bc5 2024-11-09T10:15:58,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/9dfa35bc3e7043628843d3cfffe9c2d8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/9dfa35bc3e7043628843d3cfffe9c2d8 2024-11-09T10:15:58,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/6354a920550f471aa8d7134d12c47941 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/6354a920550f471aa8d7134d12c47941 2024-11-09T10:15:58,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/ea3cfa5dc7694248a4f53bea47a974c5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/ea3cfa5dc7694248a4f53bea47a974c5 2024-11-09T10:15:58,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21d8048de5004dcda82b0b9d453b8bfe to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21d8048de5004dcda82b0b9d453b8bfe 2024-11-09T10:15:58,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3cc56ace0f80431cac4a38809f6a55cc to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/3cc56ace0f80431cac4a38809f6a55cc 2024-11-09T10:15:58,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/7b837c122aa84aac9509dabad41d8e67 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/7b837c122aa84aac9509dabad41d8e67 2024-11-09T10:15:58,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/37fd9fc3af1148feb8a4e43104d19097 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/37fd9fc3af1148feb8a4e43104d19097 2024-11-09T10:15:58,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/031590af80204de0b88708cd6624c748 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/031590af80204de0b88708cd6624c748 2024-11-09T10:15:58,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21144fb68b7a41758cf62ea10dd55280 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/21144fb68b7a41758cf62ea10dd55280 2024-11-09T10:15:58,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4abd705050de485fa886424f1a3d9d07 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4abd705050de485fa886424f1a3d9d07 2024-11-09T10:15:58,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2c49da7c373d452c97cce6171393b5ea to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/2c49da7c373d452c97cce6171393b5ea 2024-11-09T10:15:58,409 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/recovered.edits/453.seqid, newMaxSeqId=453, maxSeqId=1 2024-11-09T10:15:58,409 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. 2024-11-09T10:15:58,410 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for cac2483be99c556ef585845f10246658: Waiting for close lock at 1731147357009Running coprocessor pre-close hooks at 1731147357009Disabling compacts and flushes for region at 1731147357009Disabling writes for close at 1731147357009Obtaining lock to block concurrent updates at 1731147357010 (+1 ms)Preparing flush snapshotting stores in cac2483be99c556ef585845f10246658 at 1731147357010Finished memstore snapshotting TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731147357010Flushing stores of TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658. at 1731147357011 (+1 ms)Flushing cac2483be99c556ef585845f10246658/A: creating writer at 1731147357011Flushing cac2483be99c556ef585845f10246658/A: appending metadata at 1731147357016 (+5 ms)Flushing cac2483be99c556ef585845f10246658/A: closing flushed file at 1731147357016Flushing cac2483be99c556ef585845f10246658/B: creating writer at 1731147357430 (+414 ms)Flushing cac2483be99c556ef585845f10246658/B: appending metadata at 1731147357433 (+3 ms)Flushing cac2483be99c556ef585845f10246658/B: closing flushed file at 1731147357433Flushing cac2483be99c556ef585845f10246658/C: creating writer at 1731147357842 (+409 ms)Flushing cac2483be99c556ef585845f10246658/C: appending metadata at 1731147357844 (+2 ms)Flushing cac2483be99c556ef585845f10246658/C: closing flushed file at 1731147357844Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14433a44: reopening flushed file at 1731147358252 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eab4489: reopening flushed file at 1731147358258 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65cf2859: reopening flushed file at 1731147358265 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cac2483be99c556ef585845f10246658 in 1262ms, sequenceid=450, compaction requested=true at 1731147358271 (+6 ms)Writing region close event to WAL at 1731147358405 (+134 ms)Running coprocessor post-close hooks at 1731147358409 (+4 ms)Closed at 1731147358409 2024-11-09T10:15:58,412 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed cac2483be99c556ef585845f10246658 2024-11-09T10:15:58,413 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cac2483be99c556ef585845f10246658, regionState=CLOSED 2024-11-09T10:15:58,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:15:58,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T10:15:58,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure cac2483be99c556ef585845f10246658, server=e067b2c2c7b3,38427,1731147328756 in 1.5660 sec 2024-11-09T10:15:58,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-09T10:15:58,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cac2483be99c556ef585845f10246658, UNASSIGN in 1.5730 sec 2024-11-09T10:15:58,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T10:15:58,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5820 sec 2024-11-09T10:15:58,423 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147358423"}]},"ts":"1731147358423"} 2024-11-09T10:15:58,425 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T10:15:58,425 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T10:15:58,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5980 sec 2024-11-09T10:15:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T10:15:58,980 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:58,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T10:15:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:58,995 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T10:15:58,997 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:59,001 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658 2024-11-09T10:15:59,005 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/recovered.edits] 2024-11-09T10:15:59,009 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/76794a1475d546c6b8817df66ba13a99 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/76794a1475d546c6b8817df66ba13a99 2024-11-09T10:15:59,010 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/df7c7933105641178cbe9fed41f0811f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/df7c7933105641178cbe9fed41f0811f 2024-11-09T10:15:59,011 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/ef812304070d4d72ac9981aae2436a8e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/A/ef812304070d4d72ac9981aae2436a8e 2024-11-09T10:15:59,014 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/59c73980ff7244bc80e8f927a4dc6c28 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/59c73980ff7244bc80e8f927a4dc6c28 2024-11-09T10:15:59,015 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8673838672bf45168a113d5bd8ad06af to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8673838672bf45168a113d5bd8ad06af 2024-11-09T10:15:59,016 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8bc97c100e0d4780992743a0a37862f3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/B/8bc97c100e0d4780992743a0a37862f3 2024-11-09T10:15:59,018 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/30a1634e2fa9472880bd85870463d111 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/30a1634e2fa9472880bd85870463d111 2024-11-09T10:15:59,019 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4ffd2b0b8d124fd7a2c901e820c67e70 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/4ffd2b0b8d124fd7a2c901e820c67e70 2024-11-09T10:15:59,021 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/69abe007421b4919954b042bf3323f22 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/69abe007421b4919954b042bf3323f22 2024-11-09T10:15:59,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/be168a9bf2d74fe88400bb187ef187a7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/C/be168a9bf2d74fe88400bb187ef187a7 2024-11-09T10:15:59,024 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/recovered.edits/453.seqid to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658/recovered.edits/453.seqid 2024-11-09T10:15:59,025 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/cac2483be99c556ef585845f10246658 2024-11-09T10:15:59,025 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T10:15:59,031 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:59,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-09T10:15:59,041 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T10:15:59,074 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T10:15:59,075 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:59,076 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T10:15:59,076 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731147359076"}]},"ts":"9223372036854775807"} 2024-11-09T10:15:59,081 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T10:15:59,081 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cac2483be99c556ef585845f10246658, NAME => 'TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T10:15:59,081 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T10:15:59,081 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731147359081"}]},"ts":"9223372036854775807"} 2024-11-09T10:15:59,084 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T10:15:59,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:59,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-09T10:15:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T10:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T10:15:59,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:15:59,114 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1386229718_22 at /127.0.0.1:59384 [Waiting for operation #278] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e067b2c2c7b3:38427-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1452801290_22 at /127.0.0.1:39200 [Waiting for operation #285] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 187) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=788 (was 1350) 2024-11-09T10:15:59,124 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=785 2024-11-09T10:15:59,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:15:59,126 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:15:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T10:15:59,129 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T10:15:59,129 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:15:59,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-09T10:15:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T10:15:59,130 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T10:15:59,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741929_1105 (size=1061) 2024-11-09T10:15:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T10:15:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T10:15:59,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:15:59,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741930_1106 (size=53) 2024-11-09T10:15:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T10:15:59,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:15:59,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 342cd6b6dd55909d52716d5ae415f44c, disabling compactions & flushes 2024-11-09T10:15:59,953 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:15:59,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:15:59,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. after waiting 0 ms 2024-11-09T10:15:59,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:15:59,954 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:15:59,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 342cd6b6dd55909d52716d5ae415f44c: Waiting for close lock at 1731147359953Disabling compacts and flushes for region at 1731147359953Disabling writes for close at 1731147359954 (+1 ms)Writing region close event to WAL at 1731147359954Closed at 1731147359954 2024-11-09T10:15:59,957 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T10:15:59,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731147359957"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731147359957"}]},"ts":"1731147359957"} 2024-11-09T10:15:59,961 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T10:15:59,962 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T10:15:59,963 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147359962"}]},"ts":"1731147359962"} 2024-11-09T10:15:59,965 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T10:15:59,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, ASSIGN}] 2024-11-09T10:15:59,966 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, ASSIGN 2024-11-09T10:15:59,967 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, ASSIGN; state=OFFLINE, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=false 2024-11-09T10:16:00,118 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:00,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, ASSIGN because future has completed 2024-11-09T10:16:00,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T10:16:00,281 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:00,281 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:16:00,282 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,282 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:16:00,282 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,282 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,284 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,285 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:00,287 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 342cd6b6dd55909d52716d5ae415f44c columnFamilyName A 2024-11-09T10:16:00,287 DEBUG [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:00,288 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(327): Store=342cd6b6dd55909d52716d5ae415f44c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:00,288 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,289 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:00,290 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 342cd6b6dd55909d52716d5ae415f44c columnFamilyName B 2024-11-09T10:16:00,290 DEBUG [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:00,290 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(327): Store=342cd6b6dd55909d52716d5ae415f44c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:00,291 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,292 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:00,292 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 342cd6b6dd55909d52716d5ae415f44c columnFamilyName C 2024-11-09T10:16:00,292 DEBUG [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:00,293 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(327): Store=342cd6b6dd55909d52716d5ae415f44c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:00,293 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:00,293 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,294 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,294 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,295 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,295 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,295 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:16:00,296 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,298 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:16:00,299 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 342cd6b6dd55909d52716d5ae415f44c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74584957, jitterRate=0.1114024668931961}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:16:00,299 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:00,300 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 342cd6b6dd55909d52716d5ae415f44c: Running coprocessor pre-open hook at 1731147360282Writing region info on filesystem at 1731147360282Initializing all the Stores at 1731147360283 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147360283Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147360283Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147360284 (+1 ms)Cleaning up temporary data from old regions at 1731147360295 (+11 ms)Running coprocessor post-open hooks at 1731147360299 (+4 ms)Region opened successfully at 1731147360300 (+1 ms) 2024-11-09T10:16:00,300 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., pid=32, masterSystemTime=1731147360273 2024-11-09T10:16:00,302 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:00,302 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:00,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=OPEN, openSeqNum=2, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:00,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:16:00,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T10:16:00,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 in 186 msec 2024-11-09T10:16:00,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T10:16:00,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, ASSIGN in 343 msec 2024-11-09T10:16:00,310 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T10:16:00,310 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147360310"}]},"ts":"1731147360310"} 2024-11-09T10:16:00,312 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T10:16:00,313 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T10:16:00,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-09T10:16:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T10:16:01,279 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:01,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:01,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:01,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:01,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61274f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:01,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610ba11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:01,285 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:01,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:01,289 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T10:16:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e067b2c2c7b3,42133,1731147327850 2024-11-09T10:16:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@412d21f3 2024-11-09T10:16:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T10:16:01,291 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T10:16:01,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:16:01,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:16:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:01,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741931_1107 (size=1097) 2024-11-09T10:16:01,725 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T10:16:01,726 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T10:16:01,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:16:01,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, REOPEN/MOVE}] 2024-11-09T10:16:01,742 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, REOPEN/MOVE 2024-11-09T10:16:01,743 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:01,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, REOPEN/MOVE because future has completed 2024-11-09T10:16:01,745 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:16:01,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:16:01,898 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:01,898 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:16:01,898 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 342cd6b6dd55909d52716d5ae415f44c, disabling compactions & flushes 2024-11-09T10:16:01,898 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:01,898 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:01,898 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. after waiting 0 ms 2024-11-09T10:16:01,898 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:01,903 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T10:16:01,903 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:01,903 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 342cd6b6dd55909d52716d5ae415f44c: Waiting for close lock at 1731147361898Running coprocessor pre-close hooks at 1731147361898Disabling compacts and flushes for region at 1731147361898Disabling writes for close at 1731147361898Writing region close event to WAL at 1731147361899 (+1 ms)Running coprocessor post-close hooks at 1731147361903 (+4 ms)Closed at 1731147361903 2024-11-09T10:16:01,903 WARN [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 342cd6b6dd55909d52716d5ae415f44c to self. 2024-11-09T10:16:01,905 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:01,906 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=CLOSED 2024-11-09T10:16:01,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:16:01,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-09T10:16:01,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 in 164 msec 2024-11-09T10:16:01,911 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, REOPEN/MOVE; state=CLOSED, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=true 2024-11-09T10:16:02,062 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, REOPEN/MOVE because future has completed 2024-11-09T10:16:02,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:16:02,220 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,220 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:16:02,220 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,221 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:16:02,221 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,221 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,223 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,224 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:02,229 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 342cd6b6dd55909d52716d5ae415f44c columnFamilyName A 2024-11-09T10:16:02,231 DEBUG [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:02,232 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(327): Store=342cd6b6dd55909d52716d5ae415f44c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:02,232 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,233 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:02,233 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 342cd6b6dd55909d52716d5ae415f44c columnFamilyName B 2024-11-09T10:16:02,233 DEBUG [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:02,234 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(327): Store=342cd6b6dd55909d52716d5ae415f44c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:02,234 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,235 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:02,235 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 342cd6b6dd55909d52716d5ae415f44c columnFamilyName C 2024-11-09T10:16:02,235 DEBUG [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:02,236 INFO [StoreOpener-342cd6b6dd55909d52716d5ae415f44c-1 {}] regionserver.HStore(327): Store=342cd6b6dd55909d52716d5ae415f44c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:02,236 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,236 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,237 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,239 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,240 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,240 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,240 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:16:02,242 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,243 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 342cd6b6dd55909d52716d5ae415f44c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61355534, jitterRate=-0.08573129773139954}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:16:02,243 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,246 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 342cd6b6dd55909d52716d5ae415f44c: Running coprocessor pre-open hook at 1731147362221Writing region info on filesystem at 1731147362221Initializing all the Stores at 1731147362222 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147362222Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147362222Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147362222Cleaning up temporary data from old regions at 1731147362240 (+18 ms)Running coprocessor post-open hooks at 1731147362243 (+3 ms)Region opened successfully at 1731147362246 (+3 ms) 2024-11-09T10:16:02,247 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., pid=37, masterSystemTime=1731147362216 2024-11-09T10:16:02,249 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,249 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=OPEN, openSeqNum=5, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:16:02,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-09T10:16:02,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 in 189 msec 2024-11-09T10:16:02,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T10:16:02,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, REOPEN/MOVE in 514 msec 2024-11-09T10:16:02,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-09T10:16:02,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-09T10:16:02,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-09T10:16:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-09T10:16:02,266 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:02,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,268 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480d341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,270 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7033508b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,272 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e7d43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,273 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2de6bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,276 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eac5b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,277 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45029019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,280 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26afdce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,281 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38fbc0c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,283 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158acb51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bbd5c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3caf57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33780e86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cfb6c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,292 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228997ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,294 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eeff5bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,295 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f80d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:02,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:02,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b34074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:02,299 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17322130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:02,301 DEBUG [Thread-556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,301 DEBUG [Thread-567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,301 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:02,301 DEBUG [Thread-564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,302 DEBUG [Thread-573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-09T10:16:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,303 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T10:16:02,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,305 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:02,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:02,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,312 DEBUG [Thread-558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,320 DEBUG [Thread-571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,326 DEBUG [Thread-562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,329 DEBUG [Thread-569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,335 DEBUG [Thread-569 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,336 DEBUG [Thread-560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:16:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:16:02,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:02,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:02,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:02,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:02,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:02,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:02,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109871883ac5f07448396e98523ed082f0e_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147362349/Put/seqid=0 2024-11-09T10:16:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T10:16:02,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741932_1108 (size=17034) 2024-11-09T10:16:02,445 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:02,451 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109871883ac5f07448396e98523ed082f0e_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109871883ac5f07448396e98523ed082f0e_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:02,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/949ab5eb72fa4b808cb7cd9f3b20fac1, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:02,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731147362384, max timestamp=1731147362448Num uniques 859; 2024-11-09T10:16:02,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T10:16:02,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731147362384, max timestamp=1731147362448Num uniques 859; 2024-11-09T10:16:02,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T10:16:02,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/949ab5eb72fa4b808cb7cd9f3b20fac1 is 175, key is test_row_0/A:col10/1731147362349/Put/seqid=0 2024-11-09T10:16:02,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731147362384, max timestamp=1731147362448Num uniques 859; 2024-11-09T10:16:02,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T10:16:02,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:02,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:02,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741933_1109 (size=48135) 2024-11-09T10:16:02,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147422506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147422509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147422511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147422515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147422523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:02,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:02,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T10:16:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147422629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147422630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147422630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147422630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147422649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:02,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147422850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147422850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147422850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147422862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147422850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:02,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/949ab5eb72fa4b808cb7cd9f3b20fac1 2024-11-09T10:16:02,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:02,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T10:16:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:02,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/47a404bc4b544b6dba7384b12c813ac8 is 50, key is test_row_0/B:col10/1731147362349/Put/seqid=0 2024-11-09T10:16:03,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741934_1110 (size=9657) 2024-11-09T10:16:03,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/47a404bc4b544b6dba7384b12c813ac8 2024-11-09T10:16:03,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9fa95710569f437282ea6b2acb560b18 is 50, key is test_row_0/C:col10/1731147362349/Put/seqid=0 2024-11-09T10:16:03,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741935_1111 (size=9657) 2024-11-09T10:16:03,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147423159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147423170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147423170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147423170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147423170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T10:16:03,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9fa95710569f437282ea6b2acb560b18 2024-11-09T10:16:03,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/949ab5eb72fa4b808cb7cd9f3b20fac1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/949ab5eb72fa4b808cb7cd9f3b20fac1 2024-11-09T10:16:03,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/949ab5eb72fa4b808cb7cd9f3b20fac1, entries=250, sequenceid=14, filesize=47.0 K 2024-11-09T10:16:03,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/47a404bc4b544b6dba7384b12c813ac8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/47a404bc4b544b6dba7384b12c813ac8 2024-11-09T10:16:03,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/47a404bc4b544b6dba7384b12c813ac8, entries=100, sequenceid=14, filesize=9.4 K 2024-11-09T10:16:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9fa95710569f437282ea6b2acb560b18 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9fa95710569f437282ea6b2acb560b18 2024-11-09T10:16:03,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9fa95710569f437282ea6b2acb560b18, entries=100, sequenceid=14, filesize=9.4 K 2024-11-09T10:16:03,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 342cd6b6dd55909d52716d5ae415f44c in 1220ms, sequenceid=14, compaction requested=false 2024-11-09T10:16:03,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:03,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-09T10:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:03,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110954eb0070da4549af899ce4d0b7d83256_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147363672/Put/seqid=0 2024-11-09T10:16:03,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:03,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:03,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147423706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147423707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147423707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147423708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147423709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741936_1112 (size=14594) 2024-11-09T10:16:03,750 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:03,757 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110954eb0070da4549af899ce4d0b7d83256_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110954eb0070da4549af899ce4d0b7d83256_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:03,771 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/5c57ef4a8938420c8ceb1001bea00af0, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:03,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/5c57ef4a8938420c8ceb1001bea00af0 is 175, key is test_row_0/A:col10/1731147363672/Put/seqid=0 2024-11-09T10:16:03,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741937_1113 (size=39549) 2024-11-09T10:16:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147423819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147423819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147423819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147423819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147423836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:03,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:03,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:03,972 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T10:16:04,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147424029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147424029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147424029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147424029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147424049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:04,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:04,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,190 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/5c57ef4a8938420c8ceb1001bea00af0 2024-11-09T10:16:04,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/7415b8535944449d9006dd90296d4437 is 50, key is test_row_0/B:col10/1731147363672/Put/seqid=0 2024-11-09T10:16:04,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741938_1114 (size=12001) 2024-11-09T10:16:04,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/7415b8535944449d9006dd90296d4437 2024-11-09T10:16:04,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/2f12a58a7d094f00af4acda0ec20293f is 50, key is test_row_0/C:col10/1731147363672/Put/seqid=0 2024-11-09T10:16:04,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741939_1115 (size=12001) 2024-11-09T10:16:04,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147424349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147424349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147424349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147424349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147424369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T10:16:04,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:04,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/2f12a58a7d094f00af4acda0ec20293f 2024-11-09T10:16:04,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/5c57ef4a8938420c8ceb1001bea00af0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/5c57ef4a8938420c8ceb1001bea00af0 2024-11-09T10:16:04,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/5c57ef4a8938420c8ceb1001bea00af0, entries=200, sequenceid=50, filesize=38.6 K 2024-11-09T10:16:04,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/7415b8535944449d9006dd90296d4437 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7415b8535944449d9006dd90296d4437 2024-11-09T10:16:04,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7415b8535944449d9006dd90296d4437, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T10:16:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/2f12a58a7d094f00af4acda0ec20293f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/2f12a58a7d094f00af4acda0ec20293f 2024-11-09T10:16:04,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/2f12a58a7d094f00af4acda0ec20293f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T10:16:04,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 342cd6b6dd55909d52716d5ae415f44c in 1112ms, sequenceid=50, compaction requested=false 2024-11-09T10:16:04,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:04,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T10:16:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:04,794 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:16:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091e8cc955feca4e0c91d45a9063c9920d_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147363706/Put/seqid=0 2024-11-09T10:16:04,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741940_1116 (size=12154) 2024-11-09T10:16:04,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:04,876 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091e8cc955feca4e0c91d45a9063c9920d_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091e8cc955feca4e0c91d45a9063c9920d_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:04,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/1db5c6623ee1450ea8cd1545ae5da599, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:04,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/1db5c6623ee1450ea8cd1545ae5da599 is 175, key is test_row_0/A:col10/1731147363706/Put/seqid=0 2024-11-09T10:16:04,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:04,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:04,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741941_1117 (size=30955) 2024-11-09T10:16:04,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1731147364890, max timestamp=1731147364938Num uniques 859; 2024-11-09T10:16:04,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-09T10:16:04,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1731147364890, max timestamp=1731147364938Num uniques 859; 2024-11-09T10:16:04,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-09T10:16:04,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1731147364890, max timestamp=1731147364938Num uniques 859; 2024-11-09T10:16:04,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-09T10:16:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147424983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147424985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147424989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147424993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147424997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147425110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147425110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147425110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147425110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147425119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147425319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147425319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/1db5c6623ee1450ea8cd1545ae5da599 2024-11-09T10:16:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147425329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147425329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147425329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/1bd3afc77fc4440d87807694e93255cb is 50, key is test_row_0/B:col10/1731147363706/Put/seqid=0 2024-11-09T10:16:05,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741942_1118 (size=12001) 2024-11-09T10:16:05,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147425629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147425629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147425639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147425639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147425639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:05,787 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/1bd3afc77fc4440d87807694e93255cb 2024-11-09T10:16:05,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/0b679a7cd1cc4118ab3789e323137efb is 50, key is test_row_0/C:col10/1731147363706/Put/seqid=0 2024-11-09T10:16:05,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741943_1119 (size=12001) 2024-11-09T10:16:05,832 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/0b679a7cd1cc4118ab3789e323137efb 2024-11-09T10:16:05,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/1db5c6623ee1450ea8cd1545ae5da599 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/1db5c6623ee1450ea8cd1545ae5da599 2024-11-09T10:16:05,853 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/1db5c6623ee1450ea8cd1545ae5da599, entries=150, sequenceid=60, filesize=30.2 K 2024-11-09T10:16:05,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/1bd3afc77fc4440d87807694e93255cb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/1bd3afc77fc4440d87807694e93255cb 2024-11-09T10:16:05,867 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/1bd3afc77fc4440d87807694e93255cb, entries=150, sequenceid=60, filesize=11.7 K 2024-11-09T10:16:05,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/0b679a7cd1cc4118ab3789e323137efb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/0b679a7cd1cc4118ab3789e323137efb 2024-11-09T10:16:05,884 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/0b679a7cd1cc4118ab3789e323137efb, entries=150, sequenceid=60, filesize=11.7 K 2024-11-09T10:16:05,886 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 342cd6b6dd55909d52716d5ae415f44c in 1093ms, sequenceid=60, compaction requested=true 2024-11-09T10:16:05,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:05,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:05,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-09T10:16:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-09T10:16:05,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-09T10:16:05,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5830 sec 2024-11-09T10:16:05,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.5900 sec 2024-11-09T10:16:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:06,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=154.31 KB heapSize=407.25 KB 2024-11-09T10:16:06,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:06,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:06,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:06,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:06,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:06,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:06,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147426153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147426154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097fd7dbcfbd2842ad972882e77be64aaf_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147364992/Put/seqid=0 2024-11-09T10:16:06,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147426154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147426155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147426155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741944_1120 (size=17034) 2024-11-09T10:16:06,167 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:06,172 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097fd7dbcfbd2842ad972882e77be64aaf_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097fd7dbcfbd2842ad972882e77be64aaf_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:06,173 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/6517a4baf8354972a229d2a4dfe09d60, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:06,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/6517a4baf8354972a229d2a4dfe09d60 is 175, key is test_row_0/A:col10/1731147364992/Put/seqid=0 2024-11-09T10:16:06,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741945_1121 (size=48139) 2024-11-09T10:16:06,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147426259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147426259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147426269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147426269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147426269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T10:16:06,470 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:06,471 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-09T10:16:06,473 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T10:16:06,474 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:06,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:06,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147426469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147426469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147426479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147426479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147426479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T10:16:06,581 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/6517a4baf8354972a229d2a4dfe09d60 2024-11-09T10:16:06,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0decce01b73d46e3baab8d9d85b40015 is 50, key is test_row_0/B:col10/1731147364992/Put/seqid=0 2024-11-09T10:16:06,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741946_1122 (size=12001) 2024-11-09T10:16:06,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T10:16:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:06,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T10:16:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:06,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147426779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147426779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T10:16:06,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147426790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147426790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147426790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:06,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T10:16:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:06,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:06,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0decce01b73d46e3baab8d9d85b40015 2024-11-09T10:16:07,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9878d50f681b45ad82c49ca6ff734d29 is 50, key is test_row_0/C:col10/1731147364992/Put/seqid=0 2024-11-09T10:16:07,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741947_1123 (size=12001) 2024-11-09T10:16:07,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T10:16:07,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T10:16:07,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T10:16:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147427289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147427299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147427299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147427299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147427299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:07,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T10:16:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:07,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9878d50f681b45ad82c49ca6ff734d29 2024-11-09T10:16:07,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/6517a4baf8354972a229d2a4dfe09d60 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6517a4baf8354972a229d2a4dfe09d60 2024-11-09T10:16:07,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6517a4baf8354972a229d2a4dfe09d60, entries=250, sequenceid=93, filesize=47.0 K 2024-11-09T10:16:07,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0decce01b73d46e3baab8d9d85b40015 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0decce01b73d46e3baab8d9d85b40015 2024-11-09T10:16:07,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0decce01b73d46e3baab8d9d85b40015, entries=150, sequenceid=93, filesize=11.7 K 2024-11-09T10:16:07,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9878d50f681b45ad82c49ca6ff734d29 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9878d50f681b45ad82c49ca6ff734d29 2024-11-09T10:16:07,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9878d50f681b45ad82c49ca6ff734d29, entries=150, sequenceid=93, filesize=11.7 K 2024-11-09T10:16:07,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~427.13 KB/437376, currentSize=33.54 KB/34350 for 342cd6b6dd55909d52716d5ae415f44c in 1299ms, sequenceid=93, compaction requested=true 2024-11-09T10:16:07,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:07,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:07,449 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:07,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:07,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:07,450 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:07,451 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:07,451 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:07,451 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/A is initiating minor compaction (all files) 2024-11-09T10:16:07,451 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/B is initiating minor compaction (all files) 2024-11-09T10:16:07,451 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/A in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,451 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/B in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,451 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/949ab5eb72fa4b808cb7cd9f3b20fac1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/5c57ef4a8938420c8ceb1001bea00af0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/1db5c6623ee1450ea8cd1545ae5da599, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6517a4baf8354972a229d2a4dfe09d60] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=162.9 K 2024-11-09T10:16:07,451 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/47a404bc4b544b6dba7384b12c813ac8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7415b8535944449d9006dd90296d4437, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/1bd3afc77fc4440d87807694e93255cb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0decce01b73d46e3baab8d9d85b40015] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=44.6 K 2024-11-09T10:16:07,452 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,452 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/949ab5eb72fa4b808cb7cd9f3b20fac1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/5c57ef4a8938420c8ceb1001bea00af0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/1db5c6623ee1450ea8cd1545ae5da599, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6517a4baf8354972a229d2a4dfe09d60] 2024-11-09T10:16:07,453 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 949ab5eb72fa4b808cb7cd9f3b20fac1, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731147362311 2024-11-09T10:16:07,453 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 47a404bc4b544b6dba7384b12c813ac8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731147362331 2024-11-09T10:16:07,453 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7415b8535944449d9006dd90296d4437, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731147362480 2024-11-09T10:16:07,453 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c57ef4a8938420c8ceb1001bea00af0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731147362480 2024-11-09T10:16:07,454 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd3afc77fc4440d87807694e93255cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147363679 2024-11-09T10:16:07,454 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db5c6623ee1450ea8cd1545ae5da599, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147363679 2024-11-09T10:16:07,456 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0decce01b73d46e3baab8d9d85b40015, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731147364982 2024-11-09T10:16:07,456 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6517a4baf8354972a229d2a4dfe09d60, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731147364982 2024-11-09T10:16:07,471 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:07,486 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110976827d283496436b93576db881548dd2_342cd6b6dd55909d52716d5ae415f44c store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:07,497 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#B#compaction#105 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:07,499 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/83e7fb3263224ce39d811c277ca6583f is 50, key is test_row_0/B:col10/1731147364992/Put/seqid=0 2024-11-09T10:16:07,509 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110976827d283496436b93576db881548dd2_342cd6b6dd55909d52716d5ae415f44c, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:07,510 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110976827d283496436b93576db881548dd2_342cd6b6dd55909d52716d5ae415f44c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:07,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741949_1125 (size=12139) 2024-11-09T10:16:07,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741948_1124 (size=4469) 2024-11-09T10:16:07,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T10:16:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,565 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T10:16:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:07,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:07,574 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#A#compaction#104 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:07,575 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cf20f44231a14b7f9e15b97f4fc696d1 is 175, key is test_row_0/A:col10/1731147364992/Put/seqid=0 2024-11-09T10:16:07,578 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/83e7fb3263224ce39d811c277ca6583f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/83e7fb3263224ce39d811c277ca6583f 2024-11-09T10:16:07,587 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/B of 342cd6b6dd55909d52716d5ae415f44c into 83e7fb3263224ce39d811c277ca6583f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:07,587 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:07,588 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/B, priority=12, startTime=1731147367449; duration=0sec 2024-11-09T10:16:07,588 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:07,588 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:B 2024-11-09T10:16:07,588 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:07,590 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:07,590 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/C is initiating minor compaction (all files) 2024-11-09T10:16:07,590 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/C in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:07,591 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9fa95710569f437282ea6b2acb560b18, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/2f12a58a7d094f00af4acda0ec20293f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/0b679a7cd1cc4118ab3789e323137efb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9878d50f681b45ad82c49ca6ff734d29] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=44.6 K 2024-11-09T10:16:07,592 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fa95710569f437282ea6b2acb560b18, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731147362331 2024-11-09T10:16:07,596 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f12a58a7d094f00af4acda0ec20293f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731147362480 2024-11-09T10:16:07,602 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b679a7cd1cc4118ab3789e323137efb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147363679 2024-11-09T10:16:07,602 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9878d50f681b45ad82c49ca6ff734d29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731147364982 2024-11-09T10:16:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110946fc2b9090b4490a8ea42d2330259e7f_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147366152/Put/seqid=0 2024-11-09T10:16:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T10:16:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741950_1126 (size=31093) 2024-11-09T10:16:07,638 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#C#compaction#107 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:07,638 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/1eb7b173ddbc4bfabebd74608a977705 is 50, key is test_row_0/C:col10/1731147364992/Put/seqid=0 2024-11-09T10:16:07,644 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cf20f44231a14b7f9e15b97f4fc696d1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cf20f44231a14b7f9e15b97f4fc696d1 2024-11-09T10:16:07,661 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/A of 342cd6b6dd55909d52716d5ae415f44c into cf20f44231a14b7f9e15b97f4fc696d1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:07,661 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:07,661 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/A, priority=12, startTime=1731147367449; duration=0sec 2024-11-09T10:16:07,661 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:07,661 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:A 2024-11-09T10:16:07,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741951_1127 (size=12154) 2024-11-09T10:16:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:07,691 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110946fc2b9090b4490a8ea42d2330259e7f_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110946fc2b9090b4490a8ea42d2330259e7f_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/dd96bbfba36043d4a67c2bc926a2a7f3, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/dd96bbfba36043d4a67c2bc926a2a7f3 is 175, key is test_row_0/A:col10/1731147366152/Put/seqid=0 2024-11-09T10:16:07,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741952_1128 (size=12139) 2024-11-09T10:16:07,706 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/1eb7b173ddbc4bfabebd74608a977705 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/1eb7b173ddbc4bfabebd74608a977705 2024-11-09T10:16:07,718 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/C of 342cd6b6dd55909d52716d5ae415f44c into 1eb7b173ddbc4bfabebd74608a977705(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:07,718 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:07,718 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/C, priority=12, startTime=1731147367450; duration=0sec 2024-11-09T10:16:07,718 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:07,718 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:C 2024-11-09T10:16:07,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741953_1129 (size=30955) 2024-11-09T10:16:07,737 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/dd96bbfba36043d4a67c2bc926a2a7f3 2024-11-09T10:16:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/abb4320d78c94bbbb10c12f7cb2e1b91 is 50, key is test_row_0/B:col10/1731147366152/Put/seqid=0 2024-11-09T10:16:07,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741954_1130 (size=12001) 2024-11-09T10:16:08,169 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/abb4320d78c94bbbb10c12f7cb2e1b91 2024-11-09T10:16:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/d13d2d54d3184824bd8ff3b1c9cabdae is 50, key is test_row_0/C:col10/1731147366152/Put/seqid=0 2024-11-09T10:16:08,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741955_1131 (size=12001) 2024-11-09T10:16:08,202 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/d13d2d54d3184824bd8ff3b1c9cabdae 2024-11-09T10:16:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/dd96bbfba36043d4a67c2bc926a2a7f3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/dd96bbfba36043d4a67c2bc926a2a7f3 2024-11-09T10:16:08,221 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/dd96bbfba36043d4a67c2bc926a2a7f3, entries=150, sequenceid=101, filesize=30.2 K 2024-11-09T10:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/abb4320d78c94bbbb10c12f7cb2e1b91 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/abb4320d78c94bbbb10c12f7cb2e1b91 2024-11-09T10:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,230 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/abb4320d78c94bbbb10c12f7cb2e1b91, entries=150, sequenceid=101, filesize=11.7 K 2024-11-09T10:16:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/d13d2d54d3184824bd8ff3b1c9cabdae as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/d13d2d54d3184824bd8ff3b1c9cabdae 2024-11-09T10:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,245 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/d13d2d54d3184824bd8ff3b1c9cabdae, entries=150, sequenceid=101, filesize=11.7 K 2024-11-09T10:16:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,246 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 342cd6b6dd55909d52716d5ae415f44c in 682ms, sequenceid=101, compaction requested=false 2024-11-09T10:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-09T10:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-09T10:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T10:16:08,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-09T10:16:08,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7800 sec 2024-11-09T10:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:16:08,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:08,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:08,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:08,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:08,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dc10b367a3204a449582fc8dacbefab9_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147368354/Put/seqid=0 2024-11-09T10:16:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39334, totalHeapSize=89576, min timestamp=1731147368372, max timestamp=1731147368420Num uniques 859; 2024-11-09T10:16:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=39334, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-09T10:16:08,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731147368372, max timestamp=1731147368420Num uniques 859; 2024-11-09T10:16:08,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-09T10:16:08,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731147368372, max timestamp=1731147368420Num uniques 859; 2024-11-09T10:16:08,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-09T10:16:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147428432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147428433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147428436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147428436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147428438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741956_1132 (size=21918) 2024-11-09T10:16:08,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dc10b367a3204a449582fc8dacbefab9_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dc10b367a3204a449582fc8dacbefab9_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/975aaabd80d04e7981a00d12869f0543, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:08,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/975aaabd80d04e7981a00d12869f0543 is 175, key is test_row_0/A:col10/1731147368354/Put/seqid=0 2024-11-09T10:16:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:08,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741957_1133 (size=65323) 2024-11-09T10:16:08,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/975aaabd80d04e7981a00d12869f0543 2024-11-09T10:16:08,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/75de04711f1641c7ac7d67275017b174 is 50, key is test_row_0/B:col10/1731147368354/Put/seqid=0 2024-11-09T10:16:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147428559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147428559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147428559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147428560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147428560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741958_1134 (size=12001) 2024-11-09T10:16:08,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/75de04711f1641c7ac7d67275017b174 2024-11-09T10:16:08,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/cfaab9653fe8452dbff69370a1036f17 is 50, key is test_row_0/C:col10/1731147368354/Put/seqid=0 2024-11-09T10:16:08,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741959_1135 (size=12001) 2024-11-09T10:16:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T10:16:08,620 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:08,620 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-09T10:16:08,623 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:08,624 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:08,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T10:16:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T10:16:08,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147428769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147428769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T10:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:08,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:08,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147428779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147428780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147428780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:08,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T10:16:08,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:08,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:08,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:08,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:08,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T10:16:09,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/cfaab9653fe8452dbff69370a1036f17 2024-11-09T10:16:09,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/975aaabd80d04e7981a00d12869f0543 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/975aaabd80d04e7981a00d12869f0543 2024-11-09T10:16:09,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/975aaabd80d04e7981a00d12869f0543, entries=350, sequenceid=114, filesize=63.8 K 2024-11-09T10:16:09,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/75de04711f1641c7ac7d67275017b174 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75de04711f1641c7ac7d67275017b174 2024-11-09T10:16:09,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75de04711f1641c7ac7d67275017b174, entries=150, sequenceid=114, filesize=11.7 K 2024-11-09T10:16:09,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/cfaab9653fe8452dbff69370a1036f17 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/cfaab9653fe8452dbff69370a1036f17 2024-11-09T10:16:09,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/cfaab9653fe8452dbff69370a1036f17, entries=150, sequenceid=114, filesize=11.7 K 2024-11-09T10:16:09,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 342cd6b6dd55909d52716d5ae415f44c in 667ms, sequenceid=114, compaction requested=true 2024-11-09T10:16:09,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:09,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:09,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:09,039 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:09,039 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:09,041 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:09,041 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:09,041 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/B is initiating minor compaction (all files) 2024-11-09T10:16:09,041 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/A is initiating minor compaction (all files) 2024-11-09T10:16:09,041 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/A in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:09,041 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/B in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:09,041 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cf20f44231a14b7f9e15b97f4fc696d1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/dd96bbfba36043d4a67c2bc926a2a7f3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/975aaabd80d04e7981a00d12869f0543] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=124.4 K 2024-11-09T10:16:09,041 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/83e7fb3263224ce39d811c277ca6583f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/abb4320d78c94bbbb10c12f7cb2e1b91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75de04711f1641c7ac7d67275017b174] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=35.3 K 2024-11-09T10:16:09,041 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:09,042 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cf20f44231a14b7f9e15b97f4fc696d1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/dd96bbfba36043d4a67c2bc926a2a7f3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/975aaabd80d04e7981a00d12869f0543] 2024-11-09T10:16:09,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:09,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:09,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:09,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:09,042 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e7fb3263224ce39d811c277ca6583f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731147364982 2024-11-09T10:16:09,042 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf20f44231a14b7f9e15b97f4fc696d1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731147364982 2024-11-09T10:16:09,042 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting abb4320d78c94bbbb10c12f7cb2e1b91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731147366152 2024-11-09T10:16:09,043 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd96bbfba36043d4a67c2bc926a2a7f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731147366152 2024-11-09T10:16:09,043 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 75de04711f1641c7ac7d67275017b174, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731147368354 2024-11-09T10:16:09,043 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 975aaabd80d04e7981a00d12869f0543, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731147368348 2024-11-09T10:16:09,063 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#B#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:09,063 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/4b4b02ac20da438d853e731b4b2e710f is 50, key is test_row_0/B:col10/1731147368354/Put/seqid=0 2024-11-09T10:16:09,064 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:09,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T10:16:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:09,085 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-09T10:16:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:09,091 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109292b6e0fd9274ccb898d27929887f368_342cd6b6dd55909d52716d5ae415f44c store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:09,096 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109292b6e0fd9274ccb898d27929887f368_342cd6b6dd55909d52716d5ae415f44c, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:09,096 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109292b6e0fd9274ccb898d27929887f368_342cd6b6dd55909d52716d5ae415f44c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:09,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:09,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741960_1136 (size=12241) 2024-11-09T10:16:09,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147429100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147429098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147429101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,119 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/4b4b02ac20da438d853e731b4b2e710f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/4b4b02ac20da438d853e731b4b2e710f 2024-11-09T10:16:09,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147429111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147429110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,127 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/B of 342cd6b6dd55909d52716d5ae415f44c into 4b4b02ac20da438d853e731b4b2e710f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:09,127 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:09,127 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/B, priority=13, startTime=1731147369039; duration=0sec 2024-11-09T10:16:09,127 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:09,127 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:B 2024-11-09T10:16:09,128 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:09,129 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:09,129 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/C is initiating minor compaction (all files) 2024-11-09T10:16:09,129 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/C in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:09,129 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/1eb7b173ddbc4bfabebd74608a977705, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/d13d2d54d3184824bd8ff3b1c9cabdae, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/cfaab9653fe8452dbff69370a1036f17] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=35.3 K 2024-11-09T10:16:09,130 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb7b173ddbc4bfabebd74608a977705, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731147364982 2024-11-09T10:16:09,130 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting d13d2d54d3184824bd8ff3b1c9cabdae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731147366152 2024-11-09T10:16:09,130 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting cfaab9653fe8452dbff69370a1036f17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731147368354 2024-11-09T10:16:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110951304553f40e4875959fdc3f826a0233_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147368436/Put/seqid=0 2024-11-09T10:16:09,159 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#C#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:09,160 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/c5af9985499b4e389e8b700fead2e10e is 50, key is test_row_0/C:col10/1731147368354/Put/seqid=0 2024-11-09T10:16:09,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741961_1137 (size=4469) 2024-11-09T10:16:09,172 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#A#compaction#114 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:09,173 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/6a1a1afac6c745ee8f73d16443b4b601 is 175, key is test_row_0/A:col10/1731147368354/Put/seqid=0 2024-11-09T10:16:09,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741962_1138 (size=12304) 2024-11-09T10:16:09,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:09,213 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110951304553f40e4875959fdc3f826a0233_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110951304553f40e4875959fdc3f826a0233_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:09,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ce302f7b58d74db1a4fd4c5814729d30, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:09,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ce302f7b58d74db1a4fd4c5814729d30 is 175, key is test_row_0/A:col10/1731147368436/Put/seqid=0 2024-11-09T10:16:09,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147429219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147429219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147429219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147429229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147429229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741963_1139 (size=12241) 2024-11-09T10:16:09,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741964_1140 (size=31195) 2024-11-09T10:16:09,247 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/c5af9985499b4e389e8b700fead2e10e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/c5af9985499b4e389e8b700fead2e10e 2024-11-09T10:16:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T10:16:09,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741965_1141 (size=31105) 2024-11-09T10:16:09,256 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/C of 342cd6b6dd55909d52716d5ae415f44c into c5af9985499b4e389e8b700fead2e10e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:09,256 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:09,256 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/C, priority=13, startTime=1731147369042; duration=0sec 2024-11-09T10:16:09,256 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:09,256 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:C 2024-11-09T10:16:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147429429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147429429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147429429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147429439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147429439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,643 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/6a1a1afac6c745ee8f73d16443b4b601 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6a1a1afac6c745ee8f73d16443b4b601 2024-11-09T10:16:09,650 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/A of 342cd6b6dd55909d52716d5ae415f44c into 6a1a1afac6c745ee8f73d16443b4b601(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:09,650 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:09,650 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/A, priority=13, startTime=1731147369038; duration=0sec 2024-11-09T10:16:09,650 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:09,650 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:A 2024-11-09T10:16:09,655 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ce302f7b58d74db1a4fd4c5814729d30 2024-11-09T10:16:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/38d5989de49e4193bf756e5cc95df834 is 50, key is test_row_0/B:col10/1731147368436/Put/seqid=0 2024-11-09T10:16:09,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741966_1142 (size=12151) 2024-11-09T10:16:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147429739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147429739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147429739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147429749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147429749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T10:16:10,083 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/38d5989de49e4193bf756e5cc95df834 2024-11-09T10:16:10,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/132cd3aed44142ae951a65e6e58a7efa is 50, key is test_row_0/C:col10/1731147368436/Put/seqid=0 2024-11-09T10:16:10,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741967_1143 (size=12151) 2024-11-09T10:16:10,140 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/132cd3aed44142ae951a65e6e58a7efa 2024-11-09T10:16:10,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ce302f7b58d74db1a4fd4c5814729d30 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ce302f7b58d74db1a4fd4c5814729d30 2024-11-09T10:16:10,156 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ce302f7b58d74db1a4fd4c5814729d30, entries=150, sequenceid=140, filesize=30.4 K 2024-11-09T10:16:10,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/38d5989de49e4193bf756e5cc95df834 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/38d5989de49e4193bf756e5cc95df834 2024-11-09T10:16:10,177 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/38d5989de49e4193bf756e5cc95df834, entries=150, sequenceid=140, filesize=11.9 K 2024-11-09T10:16:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/132cd3aed44142ae951a65e6e58a7efa as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/132cd3aed44142ae951a65e6e58a7efa 2024-11-09T10:16:10,188 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/132cd3aed44142ae951a65e6e58a7efa, entries=150, sequenceid=140, filesize=11.9 K 2024-11-09T10:16:10,192 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=53.67 KB/54960 for 342cd6b6dd55909d52716d5ae415f44c in 1108ms, sequenceid=140, compaction requested=false 2024-11-09T10:16:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-09T10:16:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-09T10:16:10,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T10:16:10,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-09T10:16:10,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5770 sec 2024-11-09T10:16:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:10,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:16:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:10,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b0b7d77061c14ed5a21d90a65a8c5f1c_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147370250/Put/seqid=0 2024-11-09T10:16:10,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147430311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147430314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147430317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147430326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147430326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741968_1144 (size=12304) 2024-11-09T10:16:10,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147430429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147430440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147430441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147430449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147430452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147430639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147430649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147430650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147430669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147430669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:10,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b0b7d77061c14ed5a21d90a65a8c5f1c_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b0b7d77061c14ed5a21d90a65a8c5f1c_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:10,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/633ed2c7be654f1ba7d58b3a1f03f4ab, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:10,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/633ed2c7be654f1ba7d58b3a1f03f4ab is 175, key is test_row_0/A:col10/1731147370250/Put/seqid=0 2024-11-09T10:16:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T10:16:10,770 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:10,770 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-09T10:16:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T10:16:10,772 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:10,773 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:10,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:10,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741969_1145 (size=31105) 2024-11-09T10:16:10,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/633ed2c7be654f1ba7d58b3a1f03f4ab 2024-11-09T10:16:10,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/c2dde122ee1742b4a4036189aa37eab2 is 50, key is test_row_0/B:col10/1731147370250/Put/seqid=0 2024-11-09T10:16:10,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741970_1146 (size=12151) 2024-11-09T10:16:10,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/c2dde122ee1742b4a4036189aa37eab2 2024-11-09T10:16:10,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9c0154e2559a46759d7ee589f710fe1a is 50, key is test_row_0/C:col10/1731147370250/Put/seqid=0 2024-11-09T10:16:10,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741971_1147 (size=12151) 2024-11-09T10:16:10,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9c0154e2559a46759d7ee589f710fe1a 2024-11-09T10:16:10,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/633ed2c7be654f1ba7d58b3a1f03f4ab as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/633ed2c7be654f1ba7d58b3a1f03f4ab 2024-11-09T10:16:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T10:16:10,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/633ed2c7be654f1ba7d58b3a1f03f4ab, entries=150, sequenceid=156, filesize=30.4 K 2024-11-09T10:16:10,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/c2dde122ee1742b4a4036189aa37eab2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/c2dde122ee1742b4a4036189aa37eab2 2024-11-09T10:16:10,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:10,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:10,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:10,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/c2dde122ee1742b4a4036189aa37eab2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T10:16:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:10,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9c0154e2559a46759d7ee589f710fe1a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9c0154e2559a46759d7ee589f710fe1a 2024-11-09T10:16:10,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9c0154e2559a46759d7ee589f710fe1a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T10:16:10,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 342cd6b6dd55909d52716d5ae415f44c in 691ms, sequenceid=156, compaction requested=true 2024-11-09T10:16:10,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:10,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:10,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:10,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:10,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:16:10,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:10,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T10:16:10,943 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:10,945 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:10,945 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/A is initiating minor compaction (all files) 2024-11-09T10:16:10,945 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/A in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:10,945 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6a1a1afac6c745ee8f73d16443b4b601, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ce302f7b58d74db1a4fd4c5814729d30, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/633ed2c7be654f1ba7d58b3a1f03f4ab] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=91.2 K 2024-11-09T10:16:10,945 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:10,945 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6a1a1afac6c745ee8f73d16443b4b601, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ce302f7b58d74db1a4fd4c5814729d30, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/633ed2c7be654f1ba7d58b3a1f03f4ab] 2024-11-09T10:16:10,946 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a1a1afac6c745ee8f73d16443b4b601, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731147368354 2024-11-09T10:16:10,946 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce302f7b58d74db1a4fd4c5814729d30, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731147368418 2024-11-09T10:16:10,947 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 633ed2c7be654f1ba7d58b3a1f03f4ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147369107 2024-11-09T10:16:10,947 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:10,948 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:10,948 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/C is initiating minor compaction (all files) 2024-11-09T10:16:10,948 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/C in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:10,948 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/c5af9985499b4e389e8b700fead2e10e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/132cd3aed44142ae951a65e6e58a7efa, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9c0154e2559a46759d7ee589f710fe1a] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=35.7 K 2024-11-09T10:16:10,950 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c5af9985499b4e389e8b700fead2e10e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731147368354 2024-11-09T10:16:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:10,952 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 132cd3aed44142ae951a65e6e58a7efa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731147368418 2024-11-09T10:16:10,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T10:16:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:10,961 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c0154e2559a46759d7ee589f710fe1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147369107 2024-11-09T10:16:10,971 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:10,977 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109184ccbe49a9d4d89a3adfb20ddac9797_342cd6b6dd55909d52716d5ae415f44c store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:10,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110963bc0e7a552243ae9eb26e7fefe5071e_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147370950/Put/seqid=0 2024-11-09T10:16:10,979 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109184ccbe49a9d4d89a3adfb20ddac9797_342cd6b6dd55909d52716d5ae415f44c, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:10,980 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109184ccbe49a9d4d89a3adfb20ddac9797_342cd6b6dd55909d52716d5ae415f44c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:10,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147430975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147430980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147430983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,991 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#C#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,991 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/138f63e714104413a50822f551771bb2 is 50, key is test_row_0/C:col10/1731147370250/Put/seqid=0 2024-11-09T10:16:10,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147430989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147430989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741973_1149 (size=14794) 2024-11-09T10:16:11,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741974_1150 (size=12493) 2024-11-09T10:16:11,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741972_1148 (size=4469) 2024-11-09T10:16:11,029 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/138f63e714104413a50822f551771bb2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/138f63e714104413a50822f551771bb2 2024-11-09T10:16:11,037 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/C of 342cd6b6dd55909d52716d5ae415f44c into 138f63e714104413a50822f551771bb2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:11,037 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:11,037 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/C, priority=13, startTime=1731147370943; duration=0sec 2024-11-09T10:16:11,037 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:11,037 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:C 2024-11-09T10:16:11,038 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:11,039 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:11,039 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/B is initiating minor compaction (all files) 2024-11-09T10:16:11,039 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/B in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,040 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/4b4b02ac20da438d853e731b4b2e710f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/38d5989de49e4193bf756e5cc95df834, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/c2dde122ee1742b4a4036189aa37eab2] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=35.7 K 2024-11-09T10:16:11,040 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4b02ac20da438d853e731b4b2e710f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731147368354 2024-11-09T10:16:11,041 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d5989de49e4193bf756e5cc95df834, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731147368418 2024-11-09T10:16:11,041 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c2dde122ee1742b4a4036189aa37eab2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147369107 2024-11-09T10:16:11,053 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#B#compaction#125 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:11,054 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/b2eabc5f66cc49df9f8131bd625b2a01 is 50, key is test_row_0/B:col10/1731147370250/Put/seqid=0 2024-11-09T10:16:11,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741975_1151 (size=12493) 2024-11-09T10:16:11,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:11,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:11,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T10:16:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147431089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147431090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147431099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147431299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147431299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147431309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:11,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T10:16:11,418 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:11,422 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#A#compaction#122 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:11,422 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cb2b525014884b34bf742aafd82a004c is 175, key is test_row_0/A:col10/1731147370250/Put/seqid=0 2024-11-09T10:16:11,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110963bc0e7a552243ae9eb26e7fefe5071e_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963bc0e7a552243ae9eb26e7fefe5071e_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:11,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/79121608030e48888770da7f7e28367e, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:11,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/79121608030e48888770da7f7e28367e is 175, key is test_row_0/A:col10/1731147370950/Put/seqid=0 2024-11-09T10:16:11,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741976_1152 (size=31447) 2024-11-09T10:16:11,448 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cb2b525014884b34bf742aafd82a004c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cb2b525014884b34bf742aafd82a004c 2024-11-09T10:16:11,455 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/A of 342cd6b6dd55909d52716d5ae415f44c into cb2b525014884b34bf742aafd82a004c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:11,455 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:11,455 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/A, priority=13, startTime=1731147370943; duration=0sec 2024-11-09T10:16:11,458 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:11,458 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:A 2024-11-09T10:16:11,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741977_1153 (size=39749) 2024-11-09T10:16:11,472 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/b2eabc5f66cc49df9f8131bd625b2a01 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b2eabc5f66cc49df9f8131bd625b2a01 2024-11-09T10:16:11,483 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/B of 342cd6b6dd55909d52716d5ae415f44c into b2eabc5f66cc49df9f8131bd625b2a01(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:11,483 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:11,483 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/B, priority=13, startTime=1731147370943; duration=0sec 2024-11-09T10:16:11,483 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:11,483 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:B 2024-11-09T10:16:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147431499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147431508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:11,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:11,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147431609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147431609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147431619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:11,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:11,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:11,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/79121608030e48888770da7f7e28367e 2024-11-09T10:16:11,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/b8372fd77a7945e3ad5a358235f397e9 is 50, key is test_row_0/B:col10/1731147370950/Put/seqid=0 2024-11-09T10:16:11,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741978_1154 (size=12151) 2024-11-09T10:16:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T10:16:12,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:12,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:12,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147432119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147432129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:12,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147432129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:12,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/b8372fd77a7945e3ad5a358235f397e9 2024-11-09T10:16:12,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/59045466eac747b18349e32a01250a4e is 50, key is test_row_0/C:col10/1731147370950/Put/seqid=0 2024-11-09T10:16:12,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741979_1155 (size=12151) 2024-11-09T10:16:12,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:12,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:12,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147432519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:12,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147432519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:12,521 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:12,522 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:12,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:12,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/59045466eac747b18349e32a01250a4e 2024-11-09T10:16:12,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/79121608030e48888770da7f7e28367e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/79121608030e48888770da7f7e28367e 2024-11-09T10:16:12,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/79121608030e48888770da7f7e28367e, entries=200, sequenceid=177, filesize=38.8 K 2024-11-09T10:16:12,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/b8372fd77a7945e3ad5a358235f397e9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b8372fd77a7945e3ad5a358235f397e9 2024-11-09T10:16:12,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b8372fd77a7945e3ad5a358235f397e9, entries=150, sequenceid=177, filesize=11.9 K 2024-11-09T10:16:12,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/59045466eac747b18349e32a01250a4e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/59045466eac747b18349e32a01250a4e 2024-11-09T10:16:12,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/59045466eac747b18349e32a01250a4e, entries=150, sequenceid=177, filesize=11.9 K 2024-11-09T10:16:12,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 342cd6b6dd55909d52716d5ae415f44c in 1777ms, sequenceid=177, compaction requested=false 2024-11-09T10:16:12,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:12,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T10:16:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:12,770 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:16:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:12,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b37fa5533c7a481b90be535c9a451f6e_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147370972/Put/seqid=0 2024-11-09T10:16:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741980_1156 (size=12304) 2024-11-09T10:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T10:16:12,922 INFO [master/e067b2c2c7b3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T10:16:12,922 INFO [master/e067b2c2c7b3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T10:16:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:13,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:13,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147433164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147373157, max timestamp=1731147373171Num uniques 859; 2024-11-09T10:16:13,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:16:13,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147373157, max timestamp=1731147373171Num uniques 859; 2024-11-09T10:16:13,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147373157, max timestamp=1731147373171Num uniques 859; 2024-11-09T10:16:13,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:16:13,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:16:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:13,188 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b37fa5533c7a481b90be535c9a451f6e_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b37fa5533c7a481b90be535c9a451f6e_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cd5ffb502adf4726ab90f0127e50c11e, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:13,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cd5ffb502adf4726ab90f0127e50c11e is 175, key is test_row_0/A:col10/1731147370972/Put/seqid=0 2024-11-09T10:16:13,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741981_1157 (size=31105) 2024-11-09T10:16:13,194 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cd5ffb502adf4726ab90f0127e50c11e 2024-11-09T10:16:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/049c66866e6a4d978efb3355eb3972f4 is 50, key is test_row_0/B:col10/1731147370972/Put/seqid=0 2024-11-09T10:16:13,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741982_1158 (size=12151) 2024-11-09T10:16:13,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147433207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147433210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147433279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147433319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147433319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147433489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147433529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147433529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,609 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/049c66866e6a4d978efb3355eb3972f4 2024-11-09T10:16:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/393e45afe80a44ef9146a0663240443d is 50, key is test_row_0/C:col10/1731147370972/Put/seqid=0 2024-11-09T10:16:13,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741983_1159 (size=12151) 2024-11-09T10:16:13,639 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/393e45afe80a44ef9146a0663240443d 2024-11-09T10:16:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/cd5ffb502adf4726ab90f0127e50c11e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cd5ffb502adf4726ab90f0127e50c11e 2024-11-09T10:16:13,657 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cd5ffb502adf4726ab90f0127e50c11e, entries=150, sequenceid=191, filesize=30.4 K 2024-11-09T10:16:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/049c66866e6a4d978efb3355eb3972f4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/049c66866e6a4d978efb3355eb3972f4 2024-11-09T10:16:13,669 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/049c66866e6a4d978efb3355eb3972f4, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T10:16:13,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/393e45afe80a44ef9146a0663240443d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/393e45afe80a44ef9146a0663240443d 2024-11-09T10:16:13,682 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/393e45afe80a44ef9146a0663240443d, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T10:16:13,683 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 342cd6b6dd55909d52716d5ae415f44c in 913ms, sequenceid=191, compaction requested=true 2024-11-09T10:16:13,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:13,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:13,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-09T10:16:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-09T10:16:13,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-09T10:16:13,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9120 sec 2024-11-09T10:16:13,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.9170 sec 2024-11-09T10:16:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:13,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-09T10:16:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:13,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110949f5bbf3ea624804bf416894298a6e17_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147373209/Put/seqid=0 2024-11-09T10:16:13,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741984_1160 (size=14794) 2024-11-09T10:16:13,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147433848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147433848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147433849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147433959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147433959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:13,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147433968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147434169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147434169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147434178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,256 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:14,260 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110949f5bbf3ea624804bf416894298a6e17_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110949f5bbf3ea624804bf416894298a6e17_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:14,261 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/e3bdefda37df45cdbc4f4550abc7bbe9, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:14,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/e3bdefda37df45cdbc4f4550abc7bbe9 is 175, key is test_row_0/A:col10/1731147373209/Put/seqid=0 2024-11-09T10:16:14,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741985_1161 (size=39749) 2024-11-09T10:16:14,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147434479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147434479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147434489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147434529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,532 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147434548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:14,549 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:14,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/e3bdefda37df45cdbc4f4550abc7bbe9 2024-11-09T10:16:14,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/30f05183df6e4b4da9d54a8eaba1841e is 50, key is test_row_0/B:col10/1731147373209/Put/seqid=0 2024-11-09T10:16:14,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741986_1162 (size=12151) 2024-11-09T10:16:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T10:16:14,940 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:14,940 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-09T10:16:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T10:16:14,945 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:14,946 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:14,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147434999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:15,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147434999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:15,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147434999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T10:16:15,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/30f05183df6e4b4da9d54a8eaba1841e 2024-11-09T10:16:15,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T10:16:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/6b1a2c4527d44e83946ac8e6ae3978b7 is 50, key is test_row_0/C:col10/1731147373209/Put/seqid=0 2024-11-09T10:16:15,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741987_1163 (size=12151) 2024-11-09T10:16:15,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T10:16:15,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:15,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T10:16:15,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T10:16:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:15,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/6b1a2c4527d44e83946ac8e6ae3978b7 2024-11-09T10:16:15,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/e3bdefda37df45cdbc4f4550abc7bbe9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/e3bdefda37df45cdbc4f4550abc7bbe9 2024-11-09T10:16:15,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/e3bdefda37df45cdbc4f4550abc7bbe9, entries=200, sequenceid=226, filesize=38.8 K 2024-11-09T10:16:15,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/30f05183df6e4b4da9d54a8eaba1841e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/30f05183df6e4b4da9d54a8eaba1841e 2024-11-09T10:16:15,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/30f05183df6e4b4da9d54a8eaba1841e, entries=150, sequenceid=226, filesize=11.9 K 2024-11-09T10:16:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/6b1a2c4527d44e83946ac8e6ae3978b7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/6b1a2c4527d44e83946ac8e6ae3978b7 2024-11-09T10:16:15,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/6b1a2c4527d44e83946ac8e6ae3978b7, entries=150, sequenceid=226, filesize=11.9 K 2024-11-09T10:16:15,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 342cd6b6dd55909d52716d5ae415f44c in 1747ms, sequenceid=226, compaction requested=true 2024-11-09T10:16:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:15,549 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:16:15,549 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:15,551 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:15,551 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:15,551 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/B is initiating minor compaction (all files) 2024-11-09T10:16:15,551 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/A is initiating minor compaction (all files) 2024-11-09T10:16:15,551 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/A in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,551 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/B in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,551 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cb2b525014884b34bf742aafd82a004c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/79121608030e48888770da7f7e28367e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cd5ffb502adf4726ab90f0127e50c11e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/e3bdefda37df45cdbc4f4550abc7bbe9] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=138.7 K 2024-11-09T10:16:15,551 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,551 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b2eabc5f66cc49df9f8131bd625b2a01, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b8372fd77a7945e3ad5a358235f397e9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/049c66866e6a4d978efb3355eb3972f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/30f05183df6e4b4da9d54a8eaba1841e] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=47.8 K 2024-11-09T10:16:15,551 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cb2b525014884b34bf742aafd82a004c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/79121608030e48888770da7f7e28367e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cd5ffb502adf4726ab90f0127e50c11e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/e3bdefda37df45cdbc4f4550abc7bbe9] 2024-11-09T10:16:15,552 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2b525014884b34bf742aafd82a004c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147369107 2024-11-09T10:16:15,552 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2eabc5f66cc49df9f8131bd625b2a01, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147369107 2024-11-09T10:16:15,552 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8372fd77a7945e3ad5a358235f397e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731147370310 2024-11-09T10:16:15,552 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 79121608030e48888770da7f7e28367e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731147370277 2024-11-09T10:16:15,553 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 049c66866e6a4d978efb3355eb3972f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731147370971 2024-11-09T10:16:15,553 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting cd5ffb502adf4726ab90f0127e50c11e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731147370971 2024-11-09T10:16:15,555 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30f05183df6e4b4da9d54a8eaba1841e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731147373200 2024-11-09T10:16:15,555 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e3bdefda37df45cdbc4f4550abc7bbe9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731147373200 2024-11-09T10:16:15,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T10:16:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:15,562 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:15,566 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:15,573 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#B#compaction#134 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T10:16:15,574 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0f3bd5b41f774b6587b3b822c0b51486 is 50, key is test_row_0/B:col10/1731147373209/Put/seqid=0 2024-11-09T10:16:15,581 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109f650c0e91a554397a228c63e7bd52d06_342cd6b6dd55909d52716d5ae415f44c store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:15,583 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109f650c0e91a554397a228c63e7bd52d06_342cd6b6dd55909d52716d5ae415f44c, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:15,584 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f650c0e91a554397a228c63e7bd52d06_342cd6b6dd55909d52716d5ae415f44c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e726252f282346d697cad9fe1f9600fa_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147373847/Put/seqid=0 2024-11-09T10:16:15,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741988_1164 (size=12629) 2024-11-09T10:16:15,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741990_1166 (size=12304) 2024-11-09T10:16:15,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:15,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741989_1165 (size=4469) 2024-11-09T10:16:15,651 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e726252f282346d697cad9fe1f9600fa_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e726252f282346d697cad9fe1f9600fa_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:15,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:15,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a is 175, key is test_row_0/A:col10/1731147373847/Put/seqid=0 2024-11-09T10:16:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741991_1167 (size=31105) 2024-11-09T10:16:16,012 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0f3bd5b41f774b6587b3b822c0b51486 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0f3bd5b41f774b6587b3b822c0b51486 2024-11-09T10:16:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:16,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:16,021 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/B of 342cd6b6dd55909d52716d5ae415f44c into 0f3bd5b41f774b6587b3b822c0b51486(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:16,021 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:16,022 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/B, priority=12, startTime=1731147375549; duration=0sec 2024-11-09T10:16:16,022 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:16,022 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:B 2024-11-09T10:16:16,022 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:16,024 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:16,024 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/C is initiating minor compaction (all files) 2024-11-09T10:16:16,024 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/C in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:16,025 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/138f63e714104413a50822f551771bb2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/59045466eac747b18349e32a01250a4e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/393e45afe80a44ef9146a0663240443d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/6b1a2c4527d44e83946ac8e6ae3978b7] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=47.8 K 2024-11-09T10:16:16,025 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 138f63e714104413a50822f551771bb2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731147369107 2024-11-09T10:16:16,025 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59045466eac747b18349e32a01250a4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731147370310 2024-11-09T10:16:16,026 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 393e45afe80a44ef9146a0663240443d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731147370971 2024-11-09T10:16:16,026 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1a2c4527d44e83946ac8e6ae3978b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731147373200 2024-11-09T10:16:16,040 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#C#compaction#137 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:16,040 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/787d61db5dcf44639582a074622819ac is 50, key is test_row_0/C:col10/1731147373209/Put/seqid=0 2024-11-09T10:16:16,046 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#A#compaction#135 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:16,046 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/249a8966adcd46c087106e7be158b662 is 175, key is test_row_0/A:col10/1731147373209/Put/seqid=0 2024-11-09T10:16:16,063 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a 2024-11-09T10:16:16,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147376050, max timestamp=1731147376064Num uniques 859; 2024-11-09T10:16:16,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:16:16,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731147376050, max timestamp=1731147376064Num uniques 859; 2024-11-09T10:16:16,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:16:16,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147376050, max timestamp=1731147376064Num uniques 859; 2024-11-09T10:16:16,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:16:16,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/2088f5c920674f2b814c280b3ccfaf91 is 50, key is test_row_0/B:col10/1731147373847/Put/seqid=0 2024-11-09T10:16:16,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741992_1168 (size=12629) 2024-11-09T10:16:16,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741993_1169 (size=31583) 2024-11-09T10:16:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T10:16:16,096 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/249a8966adcd46c087106e7be158b662 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/249a8966adcd46c087106e7be158b662 2024-11-09T10:16:16,104 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/A of 342cd6b6dd55909d52716d5ae415f44c into 249a8966adcd46c087106e7be158b662(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:16,104 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:16,105 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/A, priority=12, startTime=1731147375549; duration=0sec 2024-11-09T10:16:16,105 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:16,105 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:A 2024-11-09T10:16:16,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741994_1170 (size=12151) 2024-11-09T10:16:16,124 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/2088f5c920674f2b814c280b3ccfaf91 2024-11-09T10:16:16,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147436116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147436117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147436122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/4ff3735d1bc24a1bae422821b9a78182 is 50, key is test_row_0/C:col10/1731147373847/Put/seqid=0 2024-11-09T10:16:16,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741995_1171 (size=12151) 2024-11-09T10:16:16,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147436239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147436239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147436239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147436449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147436449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147436459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,490 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/787d61db5dcf44639582a074622819ac as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/787d61db5dcf44639582a074622819ac 2024-11-09T10:16:16,496 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/C of 342cd6b6dd55909d52716d5ae415f44c into 787d61db5dcf44639582a074622819ac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:16,496 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:16,496 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/C, priority=12, startTime=1731147375549; duration=0sec 2024-11-09T10:16:16,496 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:16,496 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:C 2024-11-09T10:16:16,594 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/4ff3735d1bc24a1bae422821b9a78182 2024-11-09T10:16:16,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a 2024-11-09T10:16:16,603 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a, entries=150, sequenceid=237, filesize=30.4 K 2024-11-09T10:16:16,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/2088f5c920674f2b814c280b3ccfaf91 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/2088f5c920674f2b814c280b3ccfaf91 2024-11-09T10:16:16,609 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/2088f5c920674f2b814c280b3ccfaf91, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T10:16:16,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/4ff3735d1bc24a1bae422821b9a78182 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/4ff3735d1bc24a1bae422821b9a78182 2024-11-09T10:16:16,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/4ff3735d1bc24a1bae422821b9a78182, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T10:16:16,616 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 342cd6b6dd55909d52716d5ae415f44c in 1054ms, sequenceid=237, compaction requested=false 2024-11-09T10:16:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-09T10:16:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-09T10:16:16,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T10:16:16,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6720 sec 2024-11-09T10:16:16,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6910 sec 2024-11-09T10:16:16,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-09T10:16:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:16,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147436776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147436778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147436779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cf6e9071ae00480cadeeed820f470255_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147376759/Put/seqid=0 2024-11-09T10:16:16,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741996_1172 (size=17534) 2024-11-09T10:16:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147436889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147436889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147436889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:17,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147437098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T10:16:17,109 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:17,109 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-09T10:16:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T10:16:17,112 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:17,113 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:17,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:17,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147437109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:17,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147437109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:17,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:17,211 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cf6e9071ae00480cadeeed820f470255_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cf6e9071ae00480cadeeed820f470255_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:17,212 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/27fa31c6e1884768b7ddc6a06c4d128e, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:17,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/27fa31c6e1884768b7ddc6a06c4d128e is 175, key is test_row_0/A:col10/1731147376759/Put/seqid=0 2024-11-09T10:16:17,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741997_1173 (size=48639) 2024-11-09T10:16:17,217 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/27fa31c6e1884768b7ddc6a06c4d128e 2024-11-09T10:16:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T10:16:17,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/5c689d9f546948c4a01d0a4b38349299 is 50, key is test_row_0/B:col10/1731147376759/Put/seqid=0 2024-11-09T10:16:17,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741998_1174 (size=12301) 2024-11-09T10:16:17,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/5c689d9f546948c4a01d0a4b38349299 2024-11-09T10:16:17,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T10:16:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/875a13dbc7e9405d999418a5b53c8ba4 is 50, key is test_row_0/C:col10/1731147376759/Put/seqid=0 2024-11-09T10:16:17,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741999_1175 (size=12301) 2024-11-09T10:16:17,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147437409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:17,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T10:16:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T10:16:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147437429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147437429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:17,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T10:16:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:17,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/875a13dbc7e9405d999418a5b53c8ba4 2024-11-09T10:16:17,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/27fa31c6e1884768b7ddc6a06c4d128e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/27fa31c6e1884768b7ddc6a06c4d128e 2024-11-09T10:16:17,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/27fa31c6e1884768b7ddc6a06c4d128e, entries=250, sequenceid=276, filesize=47.5 K 2024-11-09T10:16:17,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/5c689d9f546948c4a01d0a4b38349299 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/5c689d9f546948c4a01d0a4b38349299 2024-11-09T10:16:17,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/5c689d9f546948c4a01d0a4b38349299, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T10:16:17,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/875a13dbc7e9405d999418a5b53c8ba4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/875a13dbc7e9405d999418a5b53c8ba4 2024-11-09T10:16:17,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/875a13dbc7e9405d999418a5b53c8ba4, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T10:16:17,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 342cd6b6dd55909d52716d5ae415f44c in 953ms, sequenceid=276, compaction requested=true 2024-11-09T10:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:17,713 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:17,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:17,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:17,714 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:17,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:17,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:17,715 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:17,715 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111327 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:17,715 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/B is initiating minor compaction (all files) 2024-11-09T10:16:17,715 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/A is initiating minor compaction (all files) 2024-11-09T10:16:17,715 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/B in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,715 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/A in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,715 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/249a8966adcd46c087106e7be158b662, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/27fa31c6e1884768b7ddc6a06c4d128e] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=108.7 K 2024-11-09T10:16:17,715 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0f3bd5b41f774b6587b3b822c0b51486, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/2088f5c920674f2b814c280b3ccfaf91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/5c689d9f546948c4a01d0a4b38349299] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=36.2 K 2024-11-09T10:16:17,715 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,715 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/249a8966adcd46c087106e7be158b662, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/27fa31c6e1884768b7ddc6a06c4d128e] 2024-11-09T10:16:17,715 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f3bd5b41f774b6587b3b822c0b51486, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731147373200 2024-11-09T10:16:17,715 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249a8966adcd46c087106e7be158b662, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731147373200 2024-11-09T10:16:17,716 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2088f5c920674f2b814c280b3ccfaf91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731147373820 2024-11-09T10:16:17,716 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa5a0cd2e0ac4d75bb6343ea3d49e42a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731147373820 2024-11-09T10:16:17,716 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c689d9f546948c4a01d0a4b38349299, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147376110 2024-11-09T10:16:17,716 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27fa31c6e1884768b7ddc6a06c4d128e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147376110 2024-11-09T10:16:17,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T10:16:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,724 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-09T10:16:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:17,726 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:17,736 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110995a30b716f2f48d58de69506156110ea_342cd6b6dd55909d52716d5ae415f44c store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:17,738 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T10:16:17,739 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/bada6fee33a84f41a29a33b352f39e1a is 50, key is test_row_0/B:col10/1731147376759/Put/seqid=0 2024-11-09T10:16:17,740 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110995a30b716f2f48d58de69506156110ea_342cd6b6dd55909d52716d5ae415f44c, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:17,741 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110995a30b716f2f48d58de69506156110ea_342cd6b6dd55909d52716d5ae415f44c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:17,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d1b7673bc80c464bac3902da04d922ad_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147376774/Put/seqid=0 2024-11-09T10:16:17,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742001_1177 (size=4469) 2024-11-09T10:16:17,758 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#A#compaction#143 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:17,759 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/fa6739f8926c4b30b9feb9108f5c33a7 is 175, key is test_row_0/A:col10/1731147376759/Put/seqid=0 2024-11-09T10:16:17,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742000_1176 (size=12881) 2024-11-09T10:16:17,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742002_1178 (size=12454) 2024-11-09T10:16:17,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742003_1179 (size=31835) 2024-11-09T10:16:17,806 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/fa6739f8926c4b30b9feb9108f5c33a7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/fa6739f8926c4b30b9feb9108f5c33a7 2024-11-09T10:16:17,812 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/A of 342cd6b6dd55909d52716d5ae415f44c into fa6739f8926c4b30b9feb9108f5c33a7(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:17,812 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:17,812 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/A, priority=13, startTime=1731147377713; duration=0sec 2024-11-09T10:16:17,812 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:17,812 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:A 2024-11-09T10:16:17,812 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:17,813 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:17,813 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/C is initiating minor compaction (all files) 2024-11-09T10:16:17,813 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/C in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:17,813 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/787d61db5dcf44639582a074622819ac, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/4ff3735d1bc24a1bae422821b9a78182, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/875a13dbc7e9405d999418a5b53c8ba4] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=36.2 K 2024-11-09T10:16:17,816 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787d61db5dcf44639582a074622819ac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731147373200 2024-11-09T10:16:17,816 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ff3735d1bc24a1bae422821b9a78182, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731147373820 2024-11-09T10:16:17,817 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 875a13dbc7e9405d999418a5b53c8ba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147376110 2024-11-09T10:16:17,827 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:17,827 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/b88e33f7f9c24764a0ad468b3e4f71f1 is 50, key is test_row_0/C:col10/1731147376759/Put/seqid=0 2024-11-09T10:16:17,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742004_1180 (size=12881) 2024-11-09T10:16:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:17,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:17,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731147377947, max timestamp=1731147377960Num uniques 859; 2024-11-09T10:16:17,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731147377947, max timestamp=1731147377960Num uniques 859; 2024-11-09T10:16:17,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T10:16:17,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T10:16:17,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731147377947, max timestamp=1731147377960Num uniques 859; 2024-11-09T10:16:17,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T10:16:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147438007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147438009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147438015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147438119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147438119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147438129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,168 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/bada6fee33a84f41a29a33b352f39e1a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bada6fee33a84f41a29a33b352f39e1a 2024-11-09T10:16:18,178 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/B of 342cd6b6dd55909d52716d5ae415f44c into bada6fee33a84f41a29a33b352f39e1a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:18,178 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:18,178 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/B, priority=13, startTime=1731147377713; duration=0sec 2024-11-09T10:16:18,178 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:18,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:18,178 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:B 2024-11-09T10:16:18,190 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d1b7673bc80c464bac3902da04d922ad_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d1b7673bc80c464bac3902da04d922ad_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/9c23525c2ecc464aa358220f4dacda95, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/9c23525c2ecc464aa358220f4dacda95 is 175, key is test_row_0/A:col10/1731147376774/Put/seqid=0 2024-11-09T10:16:18,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742005_1181 (size=31255) 2024-11-09T10:16:18,198 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/9c23525c2ecc464aa358220f4dacda95 2024-11-09T10:16:18,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0314634a3383447a9b777b78cc8c773b is 50, key is test_row_0/B:col10/1731147376774/Put/seqid=0 2024-11-09T10:16:18,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742006_1182 (size=12301) 2024-11-09T10:16:18,244 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/b88e33f7f9c24764a0ad468b3e4f71f1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/b88e33f7f9c24764a0ad468b3e4f71f1 2024-11-09T10:16:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T10:16:18,250 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/C of 342cd6b6dd55909d52716d5ae415f44c into b88e33f7f9c24764a0ad468b3e4f71f1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:18,250 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:18,250 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/C, priority=13, startTime=1731147377714; duration=0sec 2024-11-09T10:16:18,250 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:18,250 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:C 2024-11-09T10:16:18,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147438329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147438329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147438339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731147438569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33980 deadline: 1731147438569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,573 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,574 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0314634a3383447a9b777b78cc8c773b 2024-11-09T10:16:18,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147438639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/7881308d257e4787a607bf5e085b0da0 is 50, key is test_row_0/C:col10/1731147376774/Put/seqid=0 2024-11-09T10:16:18,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147438649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147438649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:18,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:18,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742007_1183 (size=12301) 2024-11-09T10:16:19,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/7881308d257e4787a607bf5e085b0da0 2024-11-09T10:16:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/9c23525c2ecc464aa358220f4dacda95 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9c23525c2ecc464aa358220f4dacda95 2024-11-09T10:16:19,070 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9c23525c2ecc464aa358220f4dacda95, entries=150, sequenceid=285, filesize=30.5 K 2024-11-09T10:16:19,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0314634a3383447a9b777b78cc8c773b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0314634a3383447a9b777b78cc8c773b 2024-11-09T10:16:19,076 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0314634a3383447a9b777b78cc8c773b, entries=150, sequenceid=285, filesize=12.0 K 2024-11-09T10:16:19,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/7881308d257e4787a607bf5e085b0da0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/7881308d257e4787a607bf5e085b0da0 2024-11-09T10:16:19,085 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/7881308d257e4787a607bf5e085b0da0, entries=150, sequenceid=285, filesize=12.0 K 2024-11-09T10:16:19,086 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 342cd6b6dd55909d52716d5ae415f44c in 1362ms, sequenceid=285, compaction requested=false 2024-11-09T10:16:19,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:19,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-09T10:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-09T10:16:19,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T10:16:19,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-11-09T10:16:19,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.9800 sec 2024-11-09T10:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:19,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-09T10:16:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:19,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090ac84d2f836c472c80bda738ed0503b5_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147379151/Put/seqid=0 2024-11-09T10:16:19,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147439169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147439169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742008_1184 (size=14994) 2024-11-09T10:16:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:19,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147439175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090ac84d2f836c472c80bda738ed0503b5_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090ac84d2f836c472c80bda738ed0503b5_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:19,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/16f34b4f3e714890b469f5d445117d7e, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:19,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/16f34b4f3e714890b469f5d445117d7e is 175, key is test_row_0/A:col10/1731147379151/Put/seqid=0 2024-11-09T10:16:19,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742009_1185 (size=39949) 2024-11-09T10:16:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T10:16:19,259 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:19,259 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-09T10:16:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T10:16:19,261 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:19,262 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:19,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147439279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147439289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T10:16:19,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T10:16:19,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:19,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:19,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147439489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147439499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T10:16:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T10:16:19,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/16f34b4f3e714890b469f5d445117d7e 2024-11-09T10:16:19,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/d0a8589ae7b34246b1f80b051ae5063d is 50, key is test_row_0/B:col10/1731147379151/Put/seqid=0 2024-11-09T10:16:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742010_1186 (size=12301) 2024-11-09T10:16:19,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/d0a8589ae7b34246b1f80b051ae5063d 2024-11-09T10:16:19,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/544041aa390e420b8679f3207acbea46 is 50, key is test_row_0/C:col10/1731147379151/Put/seqid=0 2024-11-09T10:16:19,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742011_1187 (size=12301) 2024-11-09T10:16:19,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/544041aa390e420b8679f3207acbea46 2024-11-09T10:16:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/16f34b4f3e714890b469f5d445117d7e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/16f34b4f3e714890b469f5d445117d7e 2024-11-09T10:16:19,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/16f34b4f3e714890b469f5d445117d7e, entries=200, sequenceid=325, filesize=39.0 K 2024-11-09T10:16:19,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/d0a8589ae7b34246b1f80b051ae5063d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/d0a8589ae7b34246b1f80b051ae5063d 2024-11-09T10:16:19,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/d0a8589ae7b34246b1f80b051ae5063d, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T10:16:19,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/544041aa390e420b8679f3207acbea46 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/544041aa390e420b8679f3207acbea46 2024-11-09T10:16:19,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/544041aa390e420b8679f3207acbea46, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T10:16:19,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 342cd6b6dd55909d52716d5ae415f44c in 494ms, sequenceid=325, compaction requested=true 2024-11-09T10:16:19,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:19,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:19,647 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:19,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:19,647 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:19,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:19,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:19,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:19,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:19,648 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:19,648 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:19,648 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/A is initiating minor compaction (all files) 2024-11-09T10:16:19,648 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/B is initiating minor compaction (all files) 2024-11-09T10:16:19,648 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/A in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,648 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/B in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,648 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/fa6739f8926c4b30b9feb9108f5c33a7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9c23525c2ecc464aa358220f4dacda95, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/16f34b4f3e714890b469f5d445117d7e] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=100.6 K 2024-11-09T10:16:19,648 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bada6fee33a84f41a29a33b352f39e1a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0314634a3383447a9b777b78cc8c773b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/d0a8589ae7b34246b1f80b051ae5063d] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=36.6 K 2024-11-09T10:16:19,648 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,648 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/fa6739f8926c4b30b9feb9108f5c33a7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9c23525c2ecc464aa358220f4dacda95, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/16f34b4f3e714890b469f5d445117d7e] 2024-11-09T10:16:19,648 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bada6fee33a84f41a29a33b352f39e1a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147376110 2024-11-09T10:16:19,648 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa6739f8926c4b30b9feb9108f5c33a7, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147376110 2024-11-09T10:16:19,649 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0314634a3383447a9b777b78cc8c773b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731147376774 2024-11-09T10:16:19,649 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c23525c2ecc464aa358220f4dacda95, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731147376774 2024-11-09T10:16:19,649 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a8589ae7b34246b1f80b051ae5063d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731147378005 2024-11-09T10:16:19,649 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f34b4f3e714890b469f5d445117d7e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731147377999 2024-11-09T10:16:19,656 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:19,658 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:19,658 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0a9b85eca40a447ab126abb581b68dcc is 50, key is test_row_0/B:col10/1731147379151/Put/seqid=0 2024-11-09T10:16:19,661 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411094d7ccf6416fa4fe1827bd74796d30cd5_342cd6b6dd55909d52716d5ae415f44c store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:19,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742012_1188 (size=12983) 2024-11-09T10:16:19,663 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411094d7ccf6416fa4fe1827bd74796d30cd5_342cd6b6dd55909d52716d5ae415f44c, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:19,663 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094d7ccf6416fa4fe1827bd74796d30cd5_342cd6b6dd55909d52716d5ae415f44c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:19,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742013_1189 (size=4469) 2024-11-09T10:16:19,669 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#A#compaction#153 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:19,669 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/a8e8988288a84480bd3fc80277591bb8 is 175, key is test_row_0/A:col10/1731147379151/Put/seqid=0 2024-11-09T10:16:19,670 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/0a9b85eca40a447ab126abb581b68dcc as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0a9b85eca40a447ab126abb581b68dcc 2024-11-09T10:16:19,675 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/B of 342cd6b6dd55909d52716d5ae415f44c into 0a9b85eca40a447ab126abb581b68dcc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:19,675 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:19,675 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/B, priority=13, startTime=1731147379647; duration=0sec 2024-11-09T10:16:19,675 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:19,675 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:B 2024-11-09T10:16:19,675 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:19,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742014_1190 (size=31937) 2024-11-09T10:16:19,681 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:19,681 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/C is initiating minor compaction (all files) 2024-11-09T10:16:19,681 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/C in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,681 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/b88e33f7f9c24764a0ad468b3e4f71f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/7881308d257e4787a607bf5e085b0da0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/544041aa390e420b8679f3207acbea46] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=36.6 K 2024-11-09T10:16:19,681 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b88e33f7f9c24764a0ad468b3e4f71f1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147376110 2024-11-09T10:16:19,682 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7881308d257e4787a607bf5e085b0da0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731147376774 2024-11-09T10:16:19,682 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 544041aa390e420b8679f3207acbea46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731147378005 2024-11-09T10:16:19,706 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#C#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:19,706 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/5ede27f6dfe54045ab0294acfe2a4192 is 50, key is test_row_0/C:col10/1731147379151/Put/seqid=0 2024-11-09T10:16:19,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742015_1191 (size=12983) 2024-11-09T10:16:19,719 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/5ede27f6dfe54045ab0294acfe2a4192 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/5ede27f6dfe54045ab0294acfe2a4192 2024-11-09T10:16:19,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T10:16:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:19,721 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T10:16:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:19,726 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/C of 342cd6b6dd55909d52716d5ae415f44c into 5ede27f6dfe54045ab0294acfe2a4192(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:19,727 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:19,727 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/C, priority=13, startTime=1731147379647; duration=0sec 2024-11-09T10:16:19,727 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:19,727 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:C 2024-11-09T10:16:19,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109674491c777ec4ddeb1c28a19cef1e71d_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147379165/Put/seqid=0 2024-11-09T10:16:19,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742016_1192 (size=12454) 2024-11-09T10:16:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:19,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:19,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731147379819, max timestamp=1731147379831Num uniques 859; 2024-11-09T10:16:19,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T10:16:19,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731147379819, max timestamp=1731147379831Num uniques 859; 2024-11-09T10:16:19,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731147379819, max timestamp=1731147379831Num uniques 859; 2024-11-09T10:16:19,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T10:16:19,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T10:16:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147439865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147439867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T10:16:19,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147439979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147439979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:20,079 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/a8e8988288a84480bd3fc80277591bb8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/a8e8988288a84480bd3fc80277591bb8 2024-11-09T10:16:20,084 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/A of 342cd6b6dd55909d52716d5ae415f44c into a8e8988288a84480bd3fc80277591bb8(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:20,084 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:20,084 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/A, priority=13, startTime=1731147379647; duration=0sec 2024-11-09T10:16:20,085 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:20,085 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:A 2024-11-09T10:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:20,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109674491c777ec4ddeb1c28a19cef1e71d_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109674491c777ec4ddeb1c28a19cef1e71d_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/acb6b41d1dec41239d72bc7a70c06a1b, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:20,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/acb6b41d1dec41239d72bc7a70c06a1b is 175, key is test_row_0/A:col10/1731147379165/Put/seqid=0 2024-11-09T10:16:20,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742017_1193 (size=31255) 2024-11-09T10:16:20,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147440189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:20,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147440189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:20,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,191 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147440189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T10:16:20,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147440499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:20,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147440499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:20,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:20,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:20,564 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/acb6b41d1dec41239d72bc7a70c06a1b 2024-11-09T10:16:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/3617f83414194975bc6495a9b5a2b663 is 50, key is test_row_0/B:col10/1731147379165/Put/seqid=0 2024-11-09T10:16:20,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742018_1194 (size=12301) 2024-11-09T10:16:20,585 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/3617f83414194975bc6495a9b5a2b663 2024-11-09T10:16:20,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/627eef7f90644b47af0afe892d01e696 is 50, key is test_row_0/C:col10/1731147379165/Put/seqid=0 2024-11-09T10:16:20,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742019_1195 (size=12301) 2024-11-09T10:16:20,614 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/627eef7f90644b47af0afe892d01e696 2024-11-09T10:16:20,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/acb6b41d1dec41239d72bc7a70c06a1b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/acb6b41d1dec41239d72bc7a70c06a1b 2024-11-09T10:16:20,624 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/acb6b41d1dec41239d72bc7a70c06a1b, entries=150, sequenceid=334, filesize=30.5 K 2024-11-09T10:16:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/3617f83414194975bc6495a9b5a2b663 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/3617f83414194975bc6495a9b5a2b663 2024-11-09T10:16:20,629 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/3617f83414194975bc6495a9b5a2b663, entries=150, sequenceid=334, filesize=12.0 K 2024-11-09T10:16:20,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/627eef7f90644b47af0afe892d01e696 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/627eef7f90644b47af0afe892d01e696 2024-11-09T10:16:20,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/627eef7f90644b47af0afe892d01e696, entries=150, sequenceid=334, filesize=12.0 K 2024-11-09T10:16:20,635 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 342cd6b6dd55909d52716d5ae415f44c in 913ms, sequenceid=334, compaction requested=false 2024-11-09T10:16:20,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:20,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:20,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-09T10:16:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-09T10:16:20,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-09T10:16:20,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3740 sec 2024-11-09T10:16:20,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.3790 sec 2024-11-09T10:16:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:21,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=161.02 KB heapSize=445.92 KB 2024-11-09T10:16:21,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147441018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110903a2df850e4249ec9f824ecd0e53ddfd_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_0/A:col10/1731147381013/Put/seqid=0 2024-11-09T10:16:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742020_1196 (size=14994) 2024-11-09T10:16:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147441021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147441129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147441139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147441339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147441349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T10:16:21,409 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:21,409 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-09T10:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T10:16:21,412 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:21,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:21,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:21,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:21,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110903a2df850e4249ec9f824ecd0e53ddfd_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110903a2df850e4249ec9f824ecd0e53ddfd_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:21,438 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/f194129fcc9149a1a2830fc12f91336a, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:21,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/f194129fcc9149a1a2830fc12f91336a is 175, key is test_row_0/A:col10/1731147381013/Put/seqid=0 2024-11-09T10:16:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742021_1197 (size=39949) 2024-11-09T10:16:21,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/f194129fcc9149a1a2830fc12f91336a 2024-11-09T10:16:21,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/7333ee3a864942e38fa55b4804cb3885 is 50, key is test_row_0/B:col10/1731147381013/Put/seqid=0 2024-11-09T10:16:21,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742022_1198 (size=12301) 2024-11-09T10:16:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T10:16:21,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T10:16:21,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:21,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:21,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:21,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147441649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147441659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:21,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T10:16:21,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:21,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:21,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:21,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T10:16:21,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/7333ee3a864942e38fa55b4804cb3885 2024-11-09T10:16:21,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T10:16:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:21,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:21,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9e460a91f2ff4815b4ede78619e52cbe is 50, key is test_row_0/C:col10/1731147381013/Put/seqid=0 2024-11-09T10:16:21,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742023_1199 (size=12301) 2024-11-09T10:16:22,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T10:16:22,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:22,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:22,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T10:16:22,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33920 deadline: 1731147442159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:22,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33972 deadline: 1731147442169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:22,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T10:16:22,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. as already flushing 2024-11-09T10:16:22,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:22,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:22,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731147442208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:22,210 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 342cd6b6dd55909d52716d5ae415f44c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:16:22,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9e460a91f2ff4815b4ede78619e52cbe 2024-11-09T10:16:22,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/f194129fcc9149a1a2830fc12f91336a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/f194129fcc9149a1a2830fc12f91336a 2024-11-09T10:16:22,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/f194129fcc9149a1a2830fc12f91336a, entries=200, sequenceid=373, filesize=39.0 K 2024-11-09T10:16:22,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/7333ee3a864942e38fa55b4804cb3885 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7333ee3a864942e38fa55b4804cb3885 2024-11-09T10:16:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7333ee3a864942e38fa55b4804cb3885, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T10:16:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/9e460a91f2ff4815b4ede78619e52cbe as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9e460a91f2ff4815b4ede78619e52cbe 2024-11-09T10:16:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9e460a91f2ff4815b4ede78619e52cbe, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T10:16:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=26.84 KB/27480 for 342cd6b6dd55909d52716d5ae415f44c in 1313ms, sequenceid=373, compaction requested=true 2024-11-09T10:16:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:22,328 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 342cd6b6dd55909d52716d5ae415f44c:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:16:22,328 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,330 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:22,330 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:22,330 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/B is initiating minor compaction (all files) 2024-11-09T10:16:22,330 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/A is initiating minor compaction (all files) 2024-11-09T10:16:22,330 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/A in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,330 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/B in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,330 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0a9b85eca40a447ab126abb581b68dcc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/3617f83414194975bc6495a9b5a2b663, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7333ee3a864942e38fa55b4804cb3885] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=36.7 K 2024-11-09T10:16:22,330 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/a8e8988288a84480bd3fc80277591bb8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/acb6b41d1dec41239d72bc7a70c06a1b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/f194129fcc9149a1a2830fc12f91336a] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=100.7 K 2024-11-09T10:16:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,330 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,330 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/a8e8988288a84480bd3fc80277591bb8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/acb6b41d1dec41239d72bc7a70c06a1b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/f194129fcc9149a1a2830fc12f91336a] 2024-11-09T10:16:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,331 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e8988288a84480bd3fc80277591bb8, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731147378005 2024-11-09T10:16:22,331 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9b85eca40a447ab126abb581b68dcc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731147378005 2024-11-09T10:16:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,332 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting acb6b41d1dec41239d72bc7a70c06a1b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731147379163 2024-11-09T10:16:22,332 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3617f83414194975bc6495a9b5a2b663, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731147379163 2024-11-09T10:16:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,333 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting f194129fcc9149a1a2830fc12f91336a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731147379843 2024-11-09T10:16:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,333 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7333ee3a864942e38fa55b4804cb3885, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731147379860 2024-11-09T10:16:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T10:16:22,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,336 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-09T10:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096fb5e5edbc754e73923406a3286dce35_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_1/A:col10/1731147381020/Put/seqid=0 2024-11-09T10:16:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,350 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#B#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:22,350 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/75473154d7f341b780c2ac3e19965042 is 50, key is test_row_0/B:col10/1731147381013/Put/seqid=0 2024-11-09T10:16:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,361 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,368 INFO [Thread-567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-567. 2024-11-09T10:16:22,368 DEBUG [Thread-567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:22,368 DEBUG [Thread-567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,368 DEBUG [Thread-567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,369 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110943414bd3c1c4488eb70e0164051542e6_342cd6b6dd55909d52716d5ae415f44c store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,372 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110943414bd3c1c4488eb70e0164051542e6_342cd6b6dd55909d52716d5ae415f44c, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:22,372 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110943414bd3c1c4488eb70e0164051542e6_342cd6b6dd55909d52716d5ae415f44c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,373 INFO [Thread-569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-569. 2024-11-09T10:16:22,373 DEBUG [Thread-569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:22,373 DEBUG [Thread-569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,373 DEBUG [Thread-569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:22,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742025_1201 (size=13085) 2024-11-09T10:16:22,380 INFO [Thread-571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-571. 2024-11-09T10:16:22,380 DEBUG [Thread-571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:22,380 DEBUG [Thread-571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,380 DEBUG [Thread-571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:22,381 INFO [Thread-573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-573. 2024-11-09T10:16:22,381 DEBUG [Thread-573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:22,381 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,381 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:22,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:22,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742024_1200 (size=9914) 2024-11-09T10:16:22,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742026_1202 (size=4469) 2024-11-09T10:16:22,385 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/75473154d7f341b780c2ac3e19965042 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75473154d7f341b780c2ac3e19965042 2024-11-09T10:16:22,396 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/B of 342cd6b6dd55909d52716d5ae415f44c into 75473154d7f341b780c2ac3e19965042(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:22,396 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:22,396 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/B, priority=13, startTime=1731147382328; duration=0sec 2024-11-09T10:16:22,396 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:22,396 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:B 2024-11-09T10:16:22,396 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:22,397 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:22,397 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 342cd6b6dd55909d52716d5ae415f44c/C is initiating minor compaction (all files) 2024-11-09T10:16:22,397 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 342cd6b6dd55909d52716d5ae415f44c/C in TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:22,397 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/5ede27f6dfe54045ab0294acfe2a4192, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/627eef7f90644b47af0afe892d01e696, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9e460a91f2ff4815b4ede78619e52cbe] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp, totalSize=36.7 K 2024-11-09T10:16:22,398 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ede27f6dfe54045ab0294acfe2a4192, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731147378005 2024-11-09T10:16:22,398 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 627eef7f90644b47af0afe892d01e696, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731147379163 2024-11-09T10:16:22,399 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e460a91f2ff4815b4ede78619e52cbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731147379860 2024-11-09T10:16:22,405 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:22,405 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/aa9acb0462424482a141abb7bcc392bf is 50, key is test_row_0/C:col10/1731147381013/Put/seqid=0 2024-11-09T10:16:22,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742027_1203 (size=13085) 2024-11-09T10:16:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T10:16:22,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:22,786 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 342cd6b6dd55909d52716d5ae415f44c#A#compaction#163 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:22,787 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/9992dc03f3f94d5ba27d332a81617067 is 175, key is test_row_0/A:col10/1731147381013/Put/seqid=0 2024-11-09T10:16:22,793 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096fb5e5edbc754e73923406a3286dce35_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096fb5e5edbc754e73923406a3286dce35_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:22,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742028_1204 (size=32039) 2024-11-09T10:16:22,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ed420c69153b43fda94ff405cb4f08d0, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:22,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ed420c69153b43fda94ff405cb4f08d0 is 175, key is test_row_1/A:col10/1731147381020/Put/seqid=0 2024-11-09T10:16:22,814 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/aa9acb0462424482a141abb7bcc392bf as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/aa9acb0462424482a141abb7bcc392bf 2024-11-09T10:16:22,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742029_1205 (size=22561) 2024-11-09T10:16:22,819 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/C of 342cd6b6dd55909d52716d5ae415f44c into aa9acb0462424482a141abb7bcc392bf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:22,819 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:22,819 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/C, priority=13, startTime=1731147382328; duration=0sec 2024-11-09T10:16:22,820 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:22,820 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:C 2024-11-09T10:16:23,180 INFO [Thread-564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-564. 2024-11-09T10:16:23,180 INFO [Thread-562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-562. 2024-11-09T10:16:23,180 DEBUG [Thread-562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:23,180 DEBUG [Thread-564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:23,180 DEBUG [Thread-562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:23,180 DEBUG [Thread-564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:23,180 DEBUG [Thread-564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:23,180 DEBUG [Thread-562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:23,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:23,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:23,201 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/9992dc03f3f94d5ba27d332a81617067 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9992dc03f3f94d5ba27d332a81617067 2024-11-09T10:16:23,208 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 342cd6b6dd55909d52716d5ae415f44c/A of 342cd6b6dd55909d52716d5ae415f44c into 9992dc03f3f94d5ba27d332a81617067(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:23,209 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:23,209 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., storeName=342cd6b6dd55909d52716d5ae415f44c/A, priority=13, startTime=1731147382328; duration=0sec 2024-11-09T10:16:23,209 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:23,209 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 342cd6b6dd55909d52716d5ae415f44c:A 2024-11-09T10:16:23,218 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ed420c69153b43fda94ff405cb4f08d0 2024-11-09T10:16:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/988ec282eb3a405aa014b0223cd5176d is 50, key is test_row_1/B:col10/1731147381020/Put/seqid=0 2024-11-09T10:16:23,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742030_1206 (size=9857) 2024-11-09T10:16:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T10:16:23,629 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/988ec282eb3a405aa014b0223cd5176d 2024-11-09T10:16:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/ab4de2d6875445529e1ad0b5eb07da8b is 50, key is test_row_1/C:col10/1731147381020/Put/seqid=0 2024-11-09T10:16:23,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742031_1207 (size=9857) 2024-11-09T10:16:24,043 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/ab4de2d6875445529e1ad0b5eb07da8b 2024-11-09T10:16:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/ed420c69153b43fda94ff405cb4f08d0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ed420c69153b43fda94ff405cb4f08d0 2024-11-09T10:16:24,052 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ed420c69153b43fda94ff405cb4f08d0, entries=100, sequenceid=380, filesize=22.0 K 2024-11-09T10:16:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/988ec282eb3a405aa014b0223cd5176d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/988ec282eb3a405aa014b0223cd5176d 2024-11-09T10:16:24,057 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/988ec282eb3a405aa014b0223cd5176d, entries=100, sequenceid=380, filesize=9.6 K 2024-11-09T10:16:24,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/ab4de2d6875445529e1ad0b5eb07da8b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/ab4de2d6875445529e1ad0b5eb07da8b 2024-11-09T10:16:24,063 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/ab4de2d6875445529e1ad0b5eb07da8b, entries=100, sequenceid=380, filesize=9.6 K 2024-11-09T10:16:24,064 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=13.42 KB/13740 for 342cd6b6dd55909d52716d5ae415f44c in 1727ms, sequenceid=380, compaction requested=false 2024-11-09T10:16:24,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 342cd6b6dd55909d52716d5ae415f44c: 2024-11-09T10:16:24,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:24,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-09T10:16:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-09T10:16:24,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T10:16:24,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6490 sec 2024-11-09T10:16:24,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6580 sec 2024-11-09T10:16:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T10:16:25,590 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:26,220 INFO [Thread-558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-558. 2024-11-09T10:16:26,220 DEBUG [Thread-558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:26,220 DEBUG [Thread-558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:26,220 DEBUG [Thread-558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:26,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:26,956 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T10:16:28,627 INFO [Thread-556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-556. 2024-11-09T10:16:28,627 DEBUG [Thread-556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:28,627 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:28,628 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:28,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:28,641 INFO [Thread-560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-560. 2024-11-09T10:16:28,641 DEBUG [Thread-560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:28,641 DEBUG [Thread-560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:28,641 DEBUG [Thread-560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:28,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:28,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T10:16:28,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-09T10:16:28,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-09T10:16:28,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-09T10:16:28,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2551 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2685 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1141 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3422 rows 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-09T10:16:28,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3394 rows 2024-11-09T10:16:28,642 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:16:28,642 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:16:28,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:28,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:28,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:28,643 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T10:16:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T10:16:28,647 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147388647"}]},"ts":"1731147388647"} 2024-11-09T10:16:28,649 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T10:16:28,649 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T10:16:28,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:16:28,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, UNASSIGN}] 2024-11-09T10:16:28,655 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, UNASSIGN 2024-11-09T10:16:28,656 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:28,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, UNASSIGN because future has completed 2024-11-09T10:16:28,661 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:16:28,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:16:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T10:16:28,815 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 342cd6b6dd55909d52716d5ae415f44c, disabling compactions & flushes 2024-11-09T10:16:28,815 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. after waiting 0 ms 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:28,815 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 342cd6b6dd55909d52716d5ae415f44c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=A 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:28,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=B 2024-11-09T10:16:28,816 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:28,816 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 342cd6b6dd55909d52716d5ae415f44c, store=C 2024-11-09T10:16:28,816 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:28,822 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099a20e583255c4cab867b69e9480001e3_342cd6b6dd55909d52716d5ae415f44c is 50, key is test_row_1/A:col10/1731147388639/Put/seqid=0 2024-11-09T10:16:28,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742032_1208 (size=9914) 2024-11-09T10:16:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T10:16:29,227 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:29,231 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099a20e583255c4cab867b69e9480001e3_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099a20e583255c4cab867b69e9480001e3_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:29,232 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/08aeacf68e0448cd8db7d47e4bdddaed, store: [table=TestAcidGuarantees family=A region=342cd6b6dd55909d52716d5ae415f44c] 2024-11-09T10:16:29,233 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/08aeacf68e0448cd8db7d47e4bdddaed is 175, key is test_row_1/A:col10/1731147388639/Put/seqid=0 2024-11-09T10:16:29,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742033_1209 (size=22561) 2024-11-09T10:16:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T10:16:29,637 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/08aeacf68e0448cd8db7d47e4bdddaed 2024-11-09T10:16:29,646 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/bb747171fa324547bad1a432546c2a9f is 50, key is test_row_1/B:col10/1731147388639/Put/seqid=0 2024-11-09T10:16:29,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742034_1210 (size=9857) 2024-11-09T10:16:29,659 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/bb747171fa324547bad1a432546c2a9f 2024-11-09T10:16:29,688 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/fbfa43cc098b4d9ba31df2dfebb70e4c is 50, key is test_row_1/C:col10/1731147388639/Put/seqid=0 2024-11-09T10:16:29,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742035_1211 (size=9857) 2024-11-09T10:16:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T10:16:30,096 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/fbfa43cc098b4d9ba31df2dfebb70e4c 2024-11-09T10:16:30,104 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/A/08aeacf68e0448cd8db7d47e4bdddaed as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/08aeacf68e0448cd8db7d47e4bdddaed 2024-11-09T10:16:30,110 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/08aeacf68e0448cd8db7d47e4bdddaed, entries=100, sequenceid=391, filesize=22.0 K 2024-11-09T10:16:30,111 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/B/bb747171fa324547bad1a432546c2a9f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bb747171fa324547bad1a432546c2a9f 2024-11-09T10:16:30,115 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bb747171fa324547bad1a432546c2a9f, entries=100, sequenceid=391, filesize=9.6 K 2024-11-09T10:16:30,116 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/.tmp/C/fbfa43cc098b4d9ba31df2dfebb70e4c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/fbfa43cc098b4d9ba31df2dfebb70e4c 2024-11-09T10:16:30,120 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/fbfa43cc098b4d9ba31df2dfebb70e4c, entries=100, sequenceid=391, filesize=9.6 K 2024-11-09T10:16:30,121 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 342cd6b6dd55909d52716d5ae415f44c in 1306ms, sequenceid=391, compaction requested=true 2024-11-09T10:16:30,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/949ab5eb72fa4b808cb7cd9f3b20fac1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/5c57ef4a8938420c8ceb1001bea00af0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/1db5c6623ee1450ea8cd1545ae5da599, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6517a4baf8354972a229d2a4dfe09d60, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cf20f44231a14b7f9e15b97f4fc696d1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/dd96bbfba36043d4a67c2bc926a2a7f3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/975aaabd80d04e7981a00d12869f0543, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6a1a1afac6c745ee8f73d16443b4b601, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ce302f7b58d74db1a4fd4c5814729d30, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cb2b525014884b34bf742aafd82a004c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/633ed2c7be654f1ba7d58b3a1f03f4ab, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/79121608030e48888770da7f7e28367e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cd5ffb502adf4726ab90f0127e50c11e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/e3bdefda37df45cdbc4f4550abc7bbe9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/249a8966adcd46c087106e7be158b662, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/27fa31c6e1884768b7ddc6a06c4d128e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/fa6739f8926c4b30b9feb9108f5c33a7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9c23525c2ecc464aa358220f4dacda95, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/16f34b4f3e714890b469f5d445117d7e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/a8e8988288a84480bd3fc80277591bb8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/acb6b41d1dec41239d72bc7a70c06a1b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/f194129fcc9149a1a2830fc12f91336a] to archive 2024-11-09T10:16:30,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:16:30,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/949ab5eb72fa4b808cb7cd9f3b20fac1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/949ab5eb72fa4b808cb7cd9f3b20fac1 2024-11-09T10:16:30,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/5c57ef4a8938420c8ceb1001bea00af0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/5c57ef4a8938420c8ceb1001bea00af0 2024-11-09T10:16:30,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/1db5c6623ee1450ea8cd1545ae5da599 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/1db5c6623ee1450ea8cd1545ae5da599 2024-11-09T10:16:30,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6517a4baf8354972a229d2a4dfe09d60 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6517a4baf8354972a229d2a4dfe09d60 2024-11-09T10:16:30,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cf20f44231a14b7f9e15b97f4fc696d1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cf20f44231a14b7f9e15b97f4fc696d1 2024-11-09T10:16:30,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/dd96bbfba36043d4a67c2bc926a2a7f3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/dd96bbfba36043d4a67c2bc926a2a7f3 2024-11-09T10:16:30,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/975aaabd80d04e7981a00d12869f0543 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/975aaabd80d04e7981a00d12869f0543 2024-11-09T10:16:30,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6a1a1afac6c745ee8f73d16443b4b601 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/6a1a1afac6c745ee8f73d16443b4b601 2024-11-09T10:16:30,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ce302f7b58d74db1a4fd4c5814729d30 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ce302f7b58d74db1a4fd4c5814729d30 2024-11-09T10:16:30,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cb2b525014884b34bf742aafd82a004c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cb2b525014884b34bf742aafd82a004c 2024-11-09T10:16:30,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/633ed2c7be654f1ba7d58b3a1f03f4ab to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/633ed2c7be654f1ba7d58b3a1f03f4ab 2024-11-09T10:16:30,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/79121608030e48888770da7f7e28367e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/79121608030e48888770da7f7e28367e 2024-11-09T10:16:30,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cd5ffb502adf4726ab90f0127e50c11e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/cd5ffb502adf4726ab90f0127e50c11e 2024-11-09T10:16:30,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/e3bdefda37df45cdbc4f4550abc7bbe9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/e3bdefda37df45cdbc4f4550abc7bbe9 2024-11-09T10:16:30,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/249a8966adcd46c087106e7be158b662 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/249a8966adcd46c087106e7be158b662 2024-11-09T10:16:30,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/aa5a0cd2e0ac4d75bb6343ea3d49e42a 2024-11-09T10:16:30,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/27fa31c6e1884768b7ddc6a06c4d128e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/27fa31c6e1884768b7ddc6a06c4d128e 2024-11-09T10:16:30,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/fa6739f8926c4b30b9feb9108f5c33a7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/fa6739f8926c4b30b9feb9108f5c33a7 2024-11-09T10:16:30,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9c23525c2ecc464aa358220f4dacda95 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9c23525c2ecc464aa358220f4dacda95 2024-11-09T10:16:30,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/16f34b4f3e714890b469f5d445117d7e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/16f34b4f3e714890b469f5d445117d7e 2024-11-09T10:16:30,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/a8e8988288a84480bd3fc80277591bb8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/a8e8988288a84480bd3fc80277591bb8 2024-11-09T10:16:30,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/acb6b41d1dec41239d72bc7a70c06a1b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/acb6b41d1dec41239d72bc7a70c06a1b 2024-11-09T10:16:30,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/f194129fcc9149a1a2830fc12f91336a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/f194129fcc9149a1a2830fc12f91336a 2024-11-09T10:16:30,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/47a404bc4b544b6dba7384b12c813ac8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7415b8535944449d9006dd90296d4437, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/1bd3afc77fc4440d87807694e93255cb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/83e7fb3263224ce39d811c277ca6583f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0decce01b73d46e3baab8d9d85b40015, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/abb4320d78c94bbbb10c12f7cb2e1b91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/4b4b02ac20da438d853e731b4b2e710f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75de04711f1641c7ac7d67275017b174, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/38d5989de49e4193bf756e5cc95df834, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b2eabc5f66cc49df9f8131bd625b2a01, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/c2dde122ee1742b4a4036189aa37eab2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b8372fd77a7945e3ad5a358235f397e9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/049c66866e6a4d978efb3355eb3972f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0f3bd5b41f774b6587b3b822c0b51486, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/30f05183df6e4b4da9d54a8eaba1841e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/2088f5c920674f2b814c280b3ccfaf91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bada6fee33a84f41a29a33b352f39e1a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/5c689d9f546948c4a01d0a4b38349299, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0314634a3383447a9b777b78cc8c773b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0a9b85eca40a447ab126abb581b68dcc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/d0a8589ae7b34246b1f80b051ae5063d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/3617f83414194975bc6495a9b5a2b663, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7333ee3a864942e38fa55b4804cb3885] to archive 2024-11-09T10:16:30,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:16:30,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/47a404bc4b544b6dba7384b12c813ac8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/47a404bc4b544b6dba7384b12c813ac8 2024-11-09T10:16:30,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7415b8535944449d9006dd90296d4437 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7415b8535944449d9006dd90296d4437 2024-11-09T10:16:30,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/1bd3afc77fc4440d87807694e93255cb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/1bd3afc77fc4440d87807694e93255cb 2024-11-09T10:16:30,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/83e7fb3263224ce39d811c277ca6583f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/83e7fb3263224ce39d811c277ca6583f 2024-11-09T10:16:30,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0decce01b73d46e3baab8d9d85b40015 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0decce01b73d46e3baab8d9d85b40015 2024-11-09T10:16:30,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/abb4320d78c94bbbb10c12f7cb2e1b91 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/abb4320d78c94bbbb10c12f7cb2e1b91 2024-11-09T10:16:30,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/4b4b02ac20da438d853e731b4b2e710f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/4b4b02ac20da438d853e731b4b2e710f 2024-11-09T10:16:30,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75de04711f1641c7ac7d67275017b174 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75de04711f1641c7ac7d67275017b174 2024-11-09T10:16:30,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/38d5989de49e4193bf756e5cc95df834 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/38d5989de49e4193bf756e5cc95df834 2024-11-09T10:16:30,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b2eabc5f66cc49df9f8131bd625b2a01 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b2eabc5f66cc49df9f8131bd625b2a01 2024-11-09T10:16:30,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/c2dde122ee1742b4a4036189aa37eab2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/c2dde122ee1742b4a4036189aa37eab2 2024-11-09T10:16:30,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b8372fd77a7945e3ad5a358235f397e9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/b8372fd77a7945e3ad5a358235f397e9 2024-11-09T10:16:30,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/049c66866e6a4d978efb3355eb3972f4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/049c66866e6a4d978efb3355eb3972f4 2024-11-09T10:16:30,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0f3bd5b41f774b6587b3b822c0b51486 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0f3bd5b41f774b6587b3b822c0b51486 2024-11-09T10:16:30,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/30f05183df6e4b4da9d54a8eaba1841e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/30f05183df6e4b4da9d54a8eaba1841e 2024-11-09T10:16:30,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/2088f5c920674f2b814c280b3ccfaf91 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/2088f5c920674f2b814c280b3ccfaf91 2024-11-09T10:16:30,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bada6fee33a84f41a29a33b352f39e1a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bada6fee33a84f41a29a33b352f39e1a 2024-11-09T10:16:30,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/5c689d9f546948c4a01d0a4b38349299 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/5c689d9f546948c4a01d0a4b38349299 2024-11-09T10:16:30,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0314634a3383447a9b777b78cc8c773b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0314634a3383447a9b777b78cc8c773b 2024-11-09T10:16:30,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0a9b85eca40a447ab126abb581b68dcc to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/0a9b85eca40a447ab126abb581b68dcc 2024-11-09T10:16:30,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/d0a8589ae7b34246b1f80b051ae5063d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/d0a8589ae7b34246b1f80b051ae5063d 2024-11-09T10:16:30,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/3617f83414194975bc6495a9b5a2b663 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/3617f83414194975bc6495a9b5a2b663 2024-11-09T10:16:30,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7333ee3a864942e38fa55b4804cb3885 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/7333ee3a864942e38fa55b4804cb3885 2024-11-09T10:16:30,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9fa95710569f437282ea6b2acb560b18, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/2f12a58a7d094f00af4acda0ec20293f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/0b679a7cd1cc4118ab3789e323137efb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/1eb7b173ddbc4bfabebd74608a977705, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9878d50f681b45ad82c49ca6ff734d29, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/d13d2d54d3184824bd8ff3b1c9cabdae, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/c5af9985499b4e389e8b700fead2e10e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/cfaab9653fe8452dbff69370a1036f17, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/132cd3aed44142ae951a65e6e58a7efa, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/138f63e714104413a50822f551771bb2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9c0154e2559a46759d7ee589f710fe1a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/59045466eac747b18349e32a01250a4e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/393e45afe80a44ef9146a0663240443d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/787d61db5dcf44639582a074622819ac, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/6b1a2c4527d44e83946ac8e6ae3978b7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/4ff3735d1bc24a1bae422821b9a78182, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/b88e33f7f9c24764a0ad468b3e4f71f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/875a13dbc7e9405d999418a5b53c8ba4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/7881308d257e4787a607bf5e085b0da0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/5ede27f6dfe54045ab0294acfe2a4192, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/544041aa390e420b8679f3207acbea46, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/627eef7f90644b47af0afe892d01e696, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9e460a91f2ff4815b4ede78619e52cbe] to archive 2024-11-09T10:16:30,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:16:30,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9fa95710569f437282ea6b2acb560b18 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9fa95710569f437282ea6b2acb560b18 2024-11-09T10:16:30,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/2f12a58a7d094f00af4acda0ec20293f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/2f12a58a7d094f00af4acda0ec20293f 2024-11-09T10:16:30,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/0b679a7cd1cc4118ab3789e323137efb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/0b679a7cd1cc4118ab3789e323137efb 2024-11-09T10:16:30,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/1eb7b173ddbc4bfabebd74608a977705 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/1eb7b173ddbc4bfabebd74608a977705 2024-11-09T10:16:30,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9878d50f681b45ad82c49ca6ff734d29 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9878d50f681b45ad82c49ca6ff734d29 2024-11-09T10:16:30,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/d13d2d54d3184824bd8ff3b1c9cabdae to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/d13d2d54d3184824bd8ff3b1c9cabdae 2024-11-09T10:16:30,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/c5af9985499b4e389e8b700fead2e10e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/c5af9985499b4e389e8b700fead2e10e 2024-11-09T10:16:30,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/cfaab9653fe8452dbff69370a1036f17 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/cfaab9653fe8452dbff69370a1036f17 2024-11-09T10:16:30,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/132cd3aed44142ae951a65e6e58a7efa to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/132cd3aed44142ae951a65e6e58a7efa 2024-11-09T10:16:30,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/138f63e714104413a50822f551771bb2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/138f63e714104413a50822f551771bb2 2024-11-09T10:16:30,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9c0154e2559a46759d7ee589f710fe1a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9c0154e2559a46759d7ee589f710fe1a 2024-11-09T10:16:30,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/59045466eac747b18349e32a01250a4e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/59045466eac747b18349e32a01250a4e 2024-11-09T10:16:30,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/393e45afe80a44ef9146a0663240443d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/393e45afe80a44ef9146a0663240443d 2024-11-09T10:16:30,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/787d61db5dcf44639582a074622819ac to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/787d61db5dcf44639582a074622819ac 2024-11-09T10:16:30,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/6b1a2c4527d44e83946ac8e6ae3978b7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/6b1a2c4527d44e83946ac8e6ae3978b7 2024-11-09T10:16:30,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/4ff3735d1bc24a1bae422821b9a78182 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/4ff3735d1bc24a1bae422821b9a78182 2024-11-09T10:16:30,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/b88e33f7f9c24764a0ad468b3e4f71f1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/b88e33f7f9c24764a0ad468b3e4f71f1 2024-11-09T10:16:30,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/875a13dbc7e9405d999418a5b53c8ba4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/875a13dbc7e9405d999418a5b53c8ba4 2024-11-09T10:16:30,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/7881308d257e4787a607bf5e085b0da0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/7881308d257e4787a607bf5e085b0da0 2024-11-09T10:16:30,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/5ede27f6dfe54045ab0294acfe2a4192 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/5ede27f6dfe54045ab0294acfe2a4192 2024-11-09T10:16:30,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/544041aa390e420b8679f3207acbea46 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/544041aa390e420b8679f3207acbea46 2024-11-09T10:16:30,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/627eef7f90644b47af0afe892d01e696 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/627eef7f90644b47af0afe892d01e696 2024-11-09T10:16:30,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9e460a91f2ff4815b4ede78619e52cbe to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/9e460a91f2ff4815b4ede78619e52cbe 2024-11-09T10:16:30,194 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/recovered.edits/394.seqid, newMaxSeqId=394, maxSeqId=4 2024-11-09T10:16:30,195 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. 2024-11-09T10:16:30,195 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 342cd6b6dd55909d52716d5ae415f44c: Waiting for close lock at 1731147388815Running coprocessor pre-close hooks at 1731147388815Disabling compacts and flushes for region at 1731147388815Disabling writes for close at 1731147388815Obtaining lock to block concurrent updates at 1731147388815Preparing flush snapshotting stores in 342cd6b6dd55909d52716d5ae415f44c at 1731147388815Finished memstore snapshotting TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731147388816 (+1 ms)Flushing stores of TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c. at 1731147388816Flushing 342cd6b6dd55909d52716d5ae415f44c/A: creating writer at 1731147388816Flushing mob file 342cd6b6dd55909d52716d5ae415f44c/A: appending metadata at 1731147388822 (+6 ms)Flushing mob file 342cd6b6dd55909d52716d5ae415f44c/A: closing flushed file at 1731147388822Flushing 342cd6b6dd55909d52716d5ae415f44c/A: appending metadata at 1731147389232 (+410 ms)Flushing 342cd6b6dd55909d52716d5ae415f44c/A: closing flushed file at 1731147389232Flushing 342cd6b6dd55909d52716d5ae415f44c/B: creating writer at 1731147389642 (+410 ms)Flushing 342cd6b6dd55909d52716d5ae415f44c/B: appending metadata at 1731147389646 (+4 ms)Flushing 342cd6b6dd55909d52716d5ae415f44c/B: closing flushed file at 1731147389646Flushing 342cd6b6dd55909d52716d5ae415f44c/C: creating writer at 1731147389666 (+20 ms)Flushing 342cd6b6dd55909d52716d5ae415f44c/C: appending metadata at 1731147389687 (+21 ms)Flushing 342cd6b6dd55909d52716d5ae415f44c/C: closing flushed file at 1731147389687Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51065afe: reopening flushed file at 1731147390103 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17c26189: reopening flushed file at 1731147390110 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b1949df: reopening flushed file at 1731147390115 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 342cd6b6dd55909d52716d5ae415f44c in 1306ms, sequenceid=391, compaction requested=true at 1731147390121 (+6 ms)Writing region close event to WAL at 1731147390191 (+70 ms)Running coprocessor post-close hooks at 1731147390195 (+4 ms)Closed at 1731147390195 2024-11-09T10:16:30,197 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=342cd6b6dd55909d52716d5ae415f44c, regionState=CLOSED 2024-11-09T10:16:30,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:16:30,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-09T10:16:30,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 342cd6b6dd55909d52716d5ae415f44c, server=e067b2c2c7b3,38427,1731147328756 in 1.5380 sec 2024-11-09T10:16:30,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-09T10:16:30,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=342cd6b6dd55909d52716d5ae415f44c, UNASSIGN in 1.5470 sec 2024-11-09T10:16:30,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-09T10:16:30,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5520 sec 2024-11-09T10:16:30,205 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147390205"}]},"ts":"1731147390205"} 2024-11-09T10:16:30,206 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T10:16:30,206 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T10:16:30,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5640 sec 2024-11-09T10:16:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T10:16:30,790 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:30,791 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T10:16:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:30,795 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T10:16:30,796 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:30,801 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,806 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/recovered.edits] 2024-11-09T10:16:30,811 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/08aeacf68e0448cd8db7d47e4bdddaed to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/08aeacf68e0448cd8db7d47e4bdddaed 2024-11-09T10:16:30,813 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9992dc03f3f94d5ba27d332a81617067 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/9992dc03f3f94d5ba27d332a81617067 2024-11-09T10:16:30,815 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ed420c69153b43fda94ff405cb4f08d0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/A/ed420c69153b43fda94ff405cb4f08d0 2024-11-09T10:16:30,818 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75473154d7f341b780c2ac3e19965042 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/75473154d7f341b780c2ac3e19965042 2024-11-09T10:16:30,820 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/988ec282eb3a405aa014b0223cd5176d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/988ec282eb3a405aa014b0223cd5176d 2024-11-09T10:16:30,822 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bb747171fa324547bad1a432546c2a9f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/B/bb747171fa324547bad1a432546c2a9f 2024-11-09T10:16:30,825 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/aa9acb0462424482a141abb7bcc392bf to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/aa9acb0462424482a141abb7bcc392bf 2024-11-09T10:16:30,827 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/ab4de2d6875445529e1ad0b5eb07da8b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/ab4de2d6875445529e1ad0b5eb07da8b 2024-11-09T10:16:30,829 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/fbfa43cc098b4d9ba31df2dfebb70e4c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/C/fbfa43cc098b4d9ba31df2dfebb70e4c 2024-11-09T10:16:30,833 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/recovered.edits/394.seqid to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c/recovered.edits/394.seqid 2024-11-09T10:16:30,834 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,834 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T10:16:30,834 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T10:16:30,836 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T10:16:30,842 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110903a2df850e4249ec9f824ecd0e53ddfd_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110903a2df850e4249ec9f824ecd0e53ddfd_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,844 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090ac84d2f836c472c80bda738ed0503b5_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090ac84d2f836c472c80bda738ed0503b5_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,846 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091e8cc955feca4e0c91d45a9063c9920d_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091e8cc955feca4e0c91d45a9063c9920d_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,847 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110946fc2b9090b4490a8ea42d2330259e7f_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110946fc2b9090b4490a8ea42d2330259e7f_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,849 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110949f5bbf3ea624804bf416894298a6e17_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110949f5bbf3ea624804bf416894298a6e17_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,851 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110951304553f40e4875959fdc3f826a0233_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110951304553f40e4875959fdc3f826a0233_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,852 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110954eb0070da4549af899ce4d0b7d83256_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110954eb0070da4549af899ce4d0b7d83256_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,854 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963bc0e7a552243ae9eb26e7fefe5071e_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963bc0e7a552243ae9eb26e7fefe5071e_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,856 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109674491c777ec4ddeb1c28a19cef1e71d_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109674491c777ec4ddeb1c28a19cef1e71d_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,858 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096fb5e5edbc754e73923406a3286dce35_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096fb5e5edbc754e73923406a3286dce35_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,860 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097fd7dbcfbd2842ad972882e77be64aaf_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097fd7dbcfbd2842ad972882e77be64aaf_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,862 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109871883ac5f07448396e98523ed082f0e_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109871883ac5f07448396e98523ed082f0e_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,864 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099a20e583255c4cab867b69e9480001e3_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099a20e583255c4cab867b69e9480001e3_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,866 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b0b7d77061c14ed5a21d90a65a8c5f1c_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b0b7d77061c14ed5a21d90a65a8c5f1c_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,867 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b37fa5533c7a481b90be535c9a451f6e_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b37fa5533c7a481b90be535c9a451f6e_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,869 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cf6e9071ae00480cadeeed820f470255_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cf6e9071ae00480cadeeed820f470255_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,870 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d1b7673bc80c464bac3902da04d922ad_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d1b7673bc80c464bac3902da04d922ad_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,872 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dc10b367a3204a449582fc8dacbefab9_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dc10b367a3204a449582fc8dacbefab9_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,873 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e726252f282346d697cad9fe1f9600fa_342cd6b6dd55909d52716d5ae415f44c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e726252f282346d697cad9fe1f9600fa_342cd6b6dd55909d52716d5ae415f44c 2024-11-09T10:16:30,873 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T10:16:30,875 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:30,877 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T10:16:30,879 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T10:16:30,880 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:30,880 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T10:16:30,881 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731147390880"}]},"ts":"9223372036854775807"} 2024-11-09T10:16:30,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T10:16:30,883 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 342cd6b6dd55909d52716d5ae415f44c, NAME => 'TestAcidGuarantees,,1731147359125.342cd6b6dd55909d52716d5ae415f44c.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T10:16:30,883 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T10:16:30,884 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731147390883"}]},"ts":"9223372036854775807"} 2024-11-09T10:16:30,886 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T10:16:30,887 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:30,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-09T10:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T10:16:30,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T10:16:30,899 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:30,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=457 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 334) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1165 (was 785) - AvailableMemoryMB LEAK? - 2024-11-09T10:16:30,918 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=1165 2024-11-09T10:16:30,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:16:30,919 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:16:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T10:16:30,921 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T10:16:30,921 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:30,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-09T10:16:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T10:16:30,922 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T10:16:30,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742036_1212 (size=1061) 2024-11-09T10:16:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T10:16:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T10:16:31,331 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:16:31,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742037_1213 (size=53) 2024-11-09T10:16:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T10:16:31,740 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:16:31,740 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aee80e883adf83e92edd542a79d1ff7e, disabling compactions & flushes 2024-11-09T10:16:31,740 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:31,740 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:31,740 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. after waiting 0 ms 2024-11-09T10:16:31,740 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:31,740 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:31,740 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aee80e883adf83e92edd542a79d1ff7e: Waiting for close lock at 1731147391740Disabling compacts and flushes for region at 1731147391740Disabling writes for close at 1731147391740Writing region close event to WAL at 1731147391740Closed at 1731147391740 2024-11-09T10:16:31,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T10:16:31,742 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731147391741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731147391741"}]},"ts":"1731147391741"} 2024-11-09T10:16:31,744 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T10:16:31,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T10:16:31,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147391747"}]},"ts":"1731147391747"} 2024-11-09T10:16:31,750 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T10:16:31,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, ASSIGN}] 2024-11-09T10:16:31,752 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, ASSIGN 2024-11-09T10:16:31,753 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, ASSIGN; state=OFFLINE, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=false 2024-11-09T10:16:31,904 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=aee80e883adf83e92edd542a79d1ff7e, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:31,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, ASSIGN because future has completed 2024-11-09T10:16:31,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:16:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T10:16:32,062 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:32,062 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:16:32,062 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,062 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:16:32,062 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,062 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,064 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,065 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:32,066 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aee80e883adf83e92edd542a79d1ff7e columnFamilyName A 2024-11-09T10:16:32,066 DEBUG [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:32,066 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.HStore(327): Store=aee80e883adf83e92edd542a79d1ff7e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:32,066 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,067 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:32,067 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aee80e883adf83e92edd542a79d1ff7e columnFamilyName B 2024-11-09T10:16:32,067 DEBUG [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:32,068 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.HStore(327): Store=aee80e883adf83e92edd542a79d1ff7e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:32,068 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,069 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:16:32,069 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aee80e883adf83e92edd542a79d1ff7e columnFamilyName C 2024-11-09T10:16:32,069 DEBUG [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:16:32,070 INFO [StoreOpener-aee80e883adf83e92edd542a79d1ff7e-1 {}] regionserver.HStore(327): Store=aee80e883adf83e92edd542a79d1ff7e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:16:32,070 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:32,070 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,071 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,071 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,072 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,072 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,072 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:16:32,074 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,076 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:16:32,076 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened aee80e883adf83e92edd542a79d1ff7e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65521659, jitterRate=-0.02365119755268097}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:16:32,076 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:32,077 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for aee80e883adf83e92edd542a79d1ff7e: Running coprocessor pre-open hook at 1731147392062Writing region info on filesystem at 1731147392062Initializing all the Stores at 1731147392063 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147392063Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147392064 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147392064Cleaning up temporary data from old regions at 1731147392072 (+8 ms)Running coprocessor post-open hooks at 1731147392076 (+4 ms)Region opened successfully at 1731147392077 (+1 ms) 2024-11-09T10:16:32,078 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., pid=61, masterSystemTime=1731147392058 2024-11-09T10:16:32,080 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:32,080 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:32,080 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=aee80e883adf83e92edd542a79d1ff7e, regionState=OPEN, openSeqNum=2, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:32,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:16:32,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T10:16:32,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 in 176 msec 2024-11-09T10:16:32,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-09T10:16:32,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, ASSIGN in 333 msec 2024-11-09T10:16:32,086 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T10:16:32,086 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147392086"}]},"ts":"1731147392086"} 2024-11-09T10:16:32,087 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T10:16:32,088 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T10:16:32,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-09T10:16:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-09T10:16:33,069 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:33,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c08f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,070 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,071 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4fa23c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,073 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b5458e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,074 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,078 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T10:16:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e067b2c2c7b3,42133,1731147327850 2024-11-09T10:16:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@423dbe50 2024-11-09T10:16:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T10:16:33,080 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T10:16:33,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd8e12d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,084 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,085 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51bbbf44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,086 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b52730d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429799b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,088 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1148cbeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,090 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e191b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480421b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,092 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f21d1bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,093 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad33f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6327b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,095 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac0f036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,096 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c51c3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fdf952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,098 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b04e5aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,099 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499e1b09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0cbf10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,101 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77797059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,102 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c368a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5735c732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,104 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4437be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,105 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b63aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212d94ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,107 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447c27c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,108 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd80f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bafd9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,109 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bb9224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,110 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29046eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b15c907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:16:33,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:16:33,112 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:16:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:16:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:16:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3815287f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:16:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:16:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:33,113 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a45055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:16:33,115 DEBUG [Thread-1023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [Thread-1014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [Thread-1012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [Thread-1027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [Thread-1018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [Thread-1021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [Thread-1010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [Thread-1029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,116 DEBUG [Thread-1016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,117 DEBUG [Thread-1025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:16:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:33,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:16:33,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:16:33,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-09T10:16:33,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:16:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T10:16:33,121 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,122 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:33,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:33,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:33,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:16:33,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:33,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:33,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:33,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:33,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:33,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:16:33,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731147393142, max timestamp=1731147393156Num uniques 859; 2024-11-09T10:16:33,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T10:16:33,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731147393142, max timestamp=1731147393156Num uniques 859; 2024-11-09T10:16:33,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T10:16:33,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731147393142, max timestamp=1731147393156Num uniques 859; 2024-11-09T10:16:33,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T10:16:33,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/1e3fe56c3c5f4afe8feea3dcce15ea3b is 50, key is test_row_0/A:col10/1731147393127/Put/seqid=0 2024-11-09T10:16:33,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742038_1214 (size=14341) 2024-11-09T10:16:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147453176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147453183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147453184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147453184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/1e3fe56c3c5f4afe8feea3dcce15ea3b 2024-11-09T10:16:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147453184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/9324799642424d84bbfab0a39a6a08df is 50, key is test_row_0/B:col10/1731147393127/Put/seqid=0 2024-11-09T10:16:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T10:16:33,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742039_1215 (size=12001) 2024-11-09T10:16:33,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T10:16:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147453289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147453289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147453298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147453299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147453299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T10:16:33,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:33,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T10:16:33,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147453498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147453498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147453509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147453509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147453509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T10:16:33,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:33,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/9324799642424d84bbfab0a39a6a08df 2024-11-09T10:16:33,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/fdbebe02f8a242aca23b5847905e5473 is 50, key is test_row_0/C:col10/1731147393127/Put/seqid=0 2024-11-09T10:16:33,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742040_1216 (size=12001) 2024-11-09T10:16:33,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T10:16:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T10:16:33,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147453809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147453809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147453819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147453819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147453819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:33,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T10:16:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:33,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:33,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:34,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T10:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:34,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:34,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/fdbebe02f8a242aca23b5847905e5473 2024-11-09T10:16:34,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/1e3fe56c3c5f4afe8feea3dcce15ea3b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1e3fe56c3c5f4afe8feea3dcce15ea3b 2024-11-09T10:16:34,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1e3fe56c3c5f4afe8feea3dcce15ea3b, entries=200, sequenceid=11, filesize=14.0 K 2024-11-09T10:16:34,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/9324799642424d84bbfab0a39a6a08df as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9324799642424d84bbfab0a39a6a08df 2024-11-09T10:16:34,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9324799642424d84bbfab0a39a6a08df, entries=150, sequenceid=11, filesize=11.7 K 2024-11-09T10:16:34,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/fdbebe02f8a242aca23b5847905e5473 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fdbebe02f8a242aca23b5847905e5473 2024-11-09T10:16:34,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fdbebe02f8a242aca23b5847905e5473, entries=150, sequenceid=11, filesize=11.7 K 2024-11-09T10:16:34,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for aee80e883adf83e92edd542a79d1ff7e in 975ms, sequenceid=11, compaction requested=false 2024-11-09T10:16:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T10:16:34,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:34,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T10:16:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:34,211 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-09T10:16:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:34,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9243540613dd4ac5bf6ab4d2fd90f174 is 50, key is test_row_0/A:col10/1731147393182/Put/seqid=0 2024-11-09T10:16:34,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742041_1217 (size=12001) 2024-11-09T10:16:34,227 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9243540613dd4ac5bf6ab4d2fd90f174 2024-11-09T10:16:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/45e58368c00f44128ecd148c75b29fa8 is 50, key is test_row_0/B:col10/1731147393182/Put/seqid=0 2024-11-09T10:16:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T10:16:34,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742042_1218 (size=12001) 2024-11-09T10:16:34,293 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/45e58368c00f44128ecd148c75b29fa8 2024-11-09T10:16:34,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2b80b527cdca4ea69508f1891e34fd38 is 50, key is test_row_0/C:col10/1731147393182/Put/seqid=0 2024-11-09T10:16:34,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742043_1219 (size=12001) 2024-11-09T10:16:34,317 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2b80b527cdca4ea69508f1891e34fd38 2024-11-09T10:16:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:34,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9243540613dd4ac5bf6ab4d2fd90f174 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9243540613dd4ac5bf6ab4d2fd90f174 2024-11-09T10:16:34,334 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9243540613dd4ac5bf6ab4d2fd90f174, entries=150, sequenceid=45, filesize=11.7 K 2024-11-09T10:16:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/45e58368c00f44128ecd148c75b29fa8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/45e58368c00f44128ecd148c75b29fa8 2024-11-09T10:16:34,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147454336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147454336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147454336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147454339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147454339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,348 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/45e58368c00f44128ecd148c75b29fa8, entries=150, sequenceid=45, filesize=11.7 K 2024-11-09T10:16:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2b80b527cdca4ea69508f1891e34fd38 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2b80b527cdca4ea69508f1891e34fd38 2024-11-09T10:16:34,369 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2b80b527cdca4ea69508f1891e34fd38, entries=150, sequenceid=45, filesize=11.7 K 2024-11-09T10:16:34,370 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for aee80e883adf83e92edd542a79d1ff7e in 159ms, sequenceid=45, compaction requested=false 2024-11-09T10:16:34,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:34,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:34,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-09T10:16:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-09T10:16:34,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-09T10:16:34,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2490 sec 2024-11-09T10:16:34,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.2590 sec 2024-11-09T10:16:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:34,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:16:34,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:34,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:34,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:34,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:34,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:34,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:34,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/fab014b60d02418f9866c505c23a6114 is 50, key is test_row_0/A:col10/1731147394451/Put/seqid=0 2024-11-09T10:16:34,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742044_1220 (size=14341) 2024-11-09T10:16:34,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147454471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147454475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147454476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147454477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147454478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147454589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147454589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147454589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147454590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147454590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147454799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147454799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147454799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147454799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147454800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:34,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/fab014b60d02418f9866c505c23a6114 2024-11-09T10:16:34,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/afd33f5e16164f938b0e9bc891a6ebad is 50, key is test_row_0/B:col10/1731147394451/Put/seqid=0 2024-11-09T10:16:34,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742045_1221 (size=12001) 2024-11-09T10:16:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147455109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147455109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147455109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147455109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147455109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T10:16:35,269 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:35,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-09T10:16:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T10:16:35,273 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:35,276 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:35,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:35,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/afd33f5e16164f938b0e9bc891a6ebad 2024-11-09T10:16:35,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df is 50, key is test_row_0/C:col10/1731147394451/Put/seqid=0 2024-11-09T10:16:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742046_1222 (size=12001) 2024-11-09T10:16:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T10:16:35,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T10:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T10:16:35,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:35,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T10:16:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147455619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147455619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147455619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147455619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147455619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:35,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df 2024-11-09T10:16:35,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/fab014b60d02418f9866c505c23a6114 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fab014b60d02418f9866c505c23a6114 2024-11-09T10:16:35,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fab014b60d02418f9866c505c23a6114, entries=200, sequenceid=59, filesize=14.0 K 2024-11-09T10:16:35,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/afd33f5e16164f938b0e9bc891a6ebad as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/afd33f5e16164f938b0e9bc891a6ebad 2024-11-09T10:16:35,722 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T10:16:35,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/afd33f5e16164f938b0e9bc891a6ebad, entries=150, sequenceid=59, filesize=11.7 K 2024-11-09T10:16:35,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df 2024-11-09T10:16:35,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T10:16:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:35,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df, entries=150, sequenceid=59, filesize=11.7 K 2024-11-09T10:16:35,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aee80e883adf83e92edd542a79d1ff7e in 1289ms, sequenceid=59, compaction requested=true 2024-11-09T10:16:35,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:35,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:35,741 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:35,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:35,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:35,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:35,741 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:35,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:35,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:35,743 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:35,743 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:35,743 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:35,743 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:35,743 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,743 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,743 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9324799642424d84bbfab0a39a6a08df, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/45e58368c00f44128ecd148c75b29fa8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/afd33f5e16164f938b0e9bc891a6ebad] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.2 K 2024-11-09T10:16:35,743 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1e3fe56c3c5f4afe8feea3dcce15ea3b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9243540613dd4ac5bf6ab4d2fd90f174, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fab014b60d02418f9866c505c23a6114] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=39.7 K 2024-11-09T10:16:35,744 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9324799642424d84bbfab0a39a6a08df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731147393127 2024-11-09T10:16:35,744 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e3fe56c3c5f4afe8feea3dcce15ea3b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731147393124 2024-11-09T10:16:35,744 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e58368c00f44128ecd148c75b29fa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731147393174 2024-11-09T10:16:35,744 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9243540613dd4ac5bf6ab4d2fd90f174, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731147393174 2024-11-09T10:16:35,744 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting afd33f5e16164f938b0e9bc891a6ebad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147394449 2024-11-09T10:16:35,745 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting fab014b60d02418f9866c505c23a6114, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147394449 2024-11-09T10:16:35,762 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:35,762 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:35,762 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d170136be5564458a7fb96871ac6c8ac is 50, key is test_row_0/A:col10/1731147394451/Put/seqid=0 2024-11-09T10:16:35,762 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/2dd48cf1a37042b7ad2d2a914c49f488 is 50, key is test_row_0/B:col10/1731147394451/Put/seqid=0 2024-11-09T10:16:35,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742047_1223 (size=12104) 2024-11-09T10:16:35,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742048_1224 (size=12104) 2024-11-09T10:16:35,775 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d170136be5564458a7fb96871ac6c8ac as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d170136be5564458a7fb96871ac6c8ac 2024-11-09T10:16:35,785 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into d170136be5564458a7fb96871ac6c8ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:35,785 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:35,785 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147395740; duration=0sec 2024-11-09T10:16:35,785 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:35,785 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:35,785 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:35,787 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:35,787 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:35,787 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,788 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fdbebe02f8a242aca23b5847905e5473, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2b80b527cdca4ea69508f1891e34fd38, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.2 K 2024-11-09T10:16:35,788 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdbebe02f8a242aca23b5847905e5473, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731147393127 2024-11-09T10:16:35,789 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b80b527cdca4ea69508f1891e34fd38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731147393174 2024-11-09T10:16:35,789 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc9b8fcb1cf48abbcb2a51a1a88c6df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147394449 2024-11-09T10:16:35,798 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:35,799 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5ba6d428b49340f3a51f8ea7bda667f6 is 50, key is test_row_0/C:col10/1731147394451/Put/seqid=0 2024-11-09T10:16:35,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742049_1225 (size=12104) 2024-11-09T10:16:35,810 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5ba6d428b49340f3a51f8ea7bda667f6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5ba6d428b49340f3a51f8ea7bda667f6 2024-11-09T10:16:35,819 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into 5ba6d428b49340f3a51f8ea7bda667f6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:35,819 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:35,819 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147395741; duration=0sec 2024-11-09T10:16:35,819 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:35,819 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:35,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T10:16:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:35,891 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T10:16:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:35,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d990449e512e4d70ab22e35a15bd8d51 is 50, key is test_row_0/A:col10/1731147394473/Put/seqid=0 2024-11-09T10:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T10:16:35,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742050_1226 (size=12001) 2024-11-09T10:16:36,178 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/2dd48cf1a37042b7ad2d2a914c49f488 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/2dd48cf1a37042b7ad2d2a914c49f488 2024-11-09T10:16:36,188 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 2dd48cf1a37042b7ad2d2a914c49f488(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:36,188 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:36,188 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147395741; duration=0sec 2024-11-09T10:16:36,188 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:36,188 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:36,309 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d990449e512e4d70ab22e35a15bd8d51 2024-11-09T10:16:36,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/b7fe074c49c84224a461692e401bea97 is 50, key is test_row_0/B:col10/1731147394473/Put/seqid=0 2024-11-09T10:16:36,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742051_1227 (size=12001) 2024-11-09T10:16:36,320 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/b7fe074c49c84224a461692e401bea97 2024-11-09T10:16:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/32c224be50ce481090b7558a79b6c9b4 is 50, key is test_row_0/C:col10/1731147394473/Put/seqid=0 2024-11-09T10:16:36,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742052_1228 (size=12001) 2024-11-09T10:16:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T10:16:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:36,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147456642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147456640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147456644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147456645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147456646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147456748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147456748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,752 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/32c224be50ce481090b7558a79b6c9b4 2024-11-09T10:16:36,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d990449e512e4d70ab22e35a15bd8d51 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d990449e512e4d70ab22e35a15bd8d51 2024-11-09T10:16:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147456759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147456759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147456759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,761 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d990449e512e4d70ab22e35a15bd8d51, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T10:16:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/b7fe074c49c84224a461692e401bea97 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b7fe074c49c84224a461692e401bea97 2024-11-09T10:16:36,766 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b7fe074c49c84224a461692e401bea97, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T10:16:36,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/32c224be50ce481090b7558a79b6c9b4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/32c224be50ce481090b7558a79b6c9b4 2024-11-09T10:16:36,771 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/32c224be50ce481090b7558a79b6c9b4, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T10:16:36,772 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for aee80e883adf83e92edd542a79d1ff7e in 881ms, sequenceid=79, compaction requested=false 2024-11-09T10:16:36,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:36,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:36,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-09T10:16:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-09T10:16:36,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T10:16:36,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-09T10:16:36,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5220 sec 2024-11-09T10:16:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:36,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T10:16:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:36,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/f64804e04b8e43798466463ae918ed2c is 50, key is test_row_0/A:col10/1731147396645/Put/seqid=0 2024-11-09T10:16:36,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742053_1229 (size=12001) 2024-11-09T10:16:36,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/f64804e04b8e43798466463ae918ed2c 2024-11-09T10:16:36,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/06917a62271c444d953e377041d22289 is 50, key is test_row_0/B:col10/1731147396645/Put/seqid=0 2024-11-09T10:16:36,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147456981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147456982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147456983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147456984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147456986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:36,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742054_1230 (size=12001) 2024-11-09T10:16:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147457098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147457098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147457099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147457099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147457099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147457308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147457308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147457309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147457309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147457309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/06917a62271c444d953e377041d22289 2024-11-09T10:16:37,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/71a888680abb4ea0b4bbe525e81e40cd is 50, key is test_row_0/C:col10/1731147396645/Put/seqid=0 2024-11-09T10:16:37,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742055_1231 (size=12001) 2024-11-09T10:16:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T10:16:37,419 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:37,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-09T10:16:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T10:16:37,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:37,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:37,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T10:16:37,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T10:16:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:37,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:37,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:37,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:37,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147457618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147457619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147457619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147457619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147457619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:37,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T10:16:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:37,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:37,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T10:16:37,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/71a888680abb4ea0b4bbe525e81e40cd 2024-11-09T10:16:37,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/f64804e04b8e43798466463ae918ed2c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/f64804e04b8e43798466463ae918ed2c 2024-11-09T10:16:37,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/f64804e04b8e43798466463ae918ed2c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T10:16:37,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/06917a62271c444d953e377041d22289 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06917a62271c444d953e377041d22289 2024-11-09T10:16:37,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06917a62271c444d953e377041d22289, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T10:16:37,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/71a888680abb4ea0b4bbe525e81e40cd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/71a888680abb4ea0b4bbe525e81e40cd 2024-11-09T10:16:37,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/71a888680abb4ea0b4bbe525e81e40cd, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T10:16:37,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for aee80e883adf83e92edd542a79d1ff7e in 870ms, sequenceid=96, compaction requested=true 2024-11-09T10:16:37,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:37,833 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:37,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:37,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:37,833 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:37,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:37,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:37,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:37,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:37,835 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:37,835 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:37,835 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:37,835 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:37,835 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:37,835 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:37,835 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/2dd48cf1a37042b7ad2d2a914c49f488, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b7fe074c49c84224a461692e401bea97, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06917a62271c444d953e377041d22289] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.3 K 2024-11-09T10:16:37,835 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d170136be5564458a7fb96871ac6c8ac, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d990449e512e4d70ab22e35a15bd8d51, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/f64804e04b8e43798466463ae918ed2c] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.3 K 2024-11-09T10:16:37,836 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting d170136be5564458a7fb96871ac6c8ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147394449 2024-11-09T10:16:37,836 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd48cf1a37042b7ad2d2a914c49f488, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147394449 2024-11-09T10:16:37,837 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting d990449e512e4d70ab22e35a15bd8d51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731147394467 2024-11-09T10:16:37,837 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b7fe074c49c84224a461692e401bea97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731147394467 2024-11-09T10:16:37,837 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 06917a62271c444d953e377041d22289, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731147396641 2024-11-09T10:16:37,837 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting f64804e04b8e43798466463ae918ed2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731147396641 2024-11-09T10:16:37,846 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:37,847 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/fa79d967df684d278f7424fa1fbedc90 is 50, key is test_row_0/A:col10/1731147396645/Put/seqid=0 2024-11-09T10:16:37,858 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:37,859 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/6aa5907b37ff4dfdb6529b8e2f385a34 is 50, key is test_row_0/B:col10/1731147396645/Put/seqid=0 2024-11-09T10:16:37,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742056_1232 (size=12207) 2024-11-09T10:16:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742057_1233 (size=12207) 2024-11-09T10:16:37,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T10:16:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:37,881 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T10:16:37,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:37,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:37,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:37,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:37,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:37,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:37,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/4a761012085746d9aa64d62be44e0d66 is 50, key is test_row_0/A:col10/1731147396980/Put/seqid=0 2024-11-09T10:16:37,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742058_1234 (size=12001) 2024-11-09T10:16:37,892 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/4a761012085746d9aa64d62be44e0d66 2024-11-09T10:16:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/ffd671b032554b1aa5f82b5528343787 is 50, key is test_row_0/B:col10/1731147396980/Put/seqid=0 2024-11-09T10:16:37,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742059_1235 (size=12001) 2024-11-09T10:16:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T10:16:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:38,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147458154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147458153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147458161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147458163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147458165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147458269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147458269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,276 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/fa79d967df684d278f7424fa1fbedc90 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fa79d967df684d278f7424fa1fbedc90 2024-11-09T10:16:38,281 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/6aa5907b37ff4dfdb6529b8e2f385a34 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/6aa5907b37ff4dfdb6529b8e2f385a34 2024-11-09T10:16:38,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147458279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147458279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147458279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,290 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into fa79d967df684d278f7424fa1fbedc90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:38,290 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:38,290 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147397832; duration=0sec 2024-11-09T10:16:38,290 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:38,290 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:38,291 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:38,295 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 6aa5907b37ff4dfdb6529b8e2f385a34(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:38,295 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:38,295 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147397833; duration=0sec 2024-11-09T10:16:38,296 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:38,296 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:38,296 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:38,296 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:38,296 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:38,296 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5ba6d428b49340f3a51f8ea7bda667f6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/32c224be50ce481090b7558a79b6c9b4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/71a888680abb4ea0b4bbe525e81e40cd] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.3 K 2024-11-09T10:16:38,297 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba6d428b49340f3a51f8ea7bda667f6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731147394449 2024-11-09T10:16:38,297 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32c224be50ce481090b7558a79b6c9b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731147394467 2024-11-09T10:16:38,297 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71a888680abb4ea0b4bbe525e81e40cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731147396641 2024-11-09T10:16:38,308 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/ffd671b032554b1aa5f82b5528343787 2024-11-09T10:16:38,308 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:38,309 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/f321c71582ad412aab2e9a2722845b97 is 50, key is test_row_0/C:col10/1731147396645/Put/seqid=0 2024-11-09T10:16:38,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5b831444325a4f5a98e77d1311690a45 is 50, key is test_row_0/C:col10/1731147396980/Put/seqid=0 2024-11-09T10:16:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742061_1237 (size=12001) 2024-11-09T10:16:38,336 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5b831444325a4f5a98e77d1311690a45 2024-11-09T10:16:38,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742060_1236 (size=12207) 2024-11-09T10:16:38,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/4a761012085746d9aa64d62be44e0d66 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4a761012085746d9aa64d62be44e0d66 2024-11-09T10:16:38,350 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/f321c71582ad412aab2e9a2722845b97 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/f321c71582ad412aab2e9a2722845b97 2024-11-09T10:16:38,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4a761012085746d9aa64d62be44e0d66, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T10:16:38,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/ffd671b032554b1aa5f82b5528343787 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/ffd671b032554b1aa5f82b5528343787 2024-11-09T10:16:38,372 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into f321c71582ad412aab2e9a2722845b97(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:38,372 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:38,372 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147397833; duration=0sec 2024-11-09T10:16:38,372 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:38,373 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:38,373 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/ffd671b032554b1aa5f82b5528343787, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T10:16:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5b831444325a4f5a98e77d1311690a45 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5b831444325a4f5a98e77d1311690a45 2024-11-09T10:16:38,383 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5b831444325a4f5a98e77d1311690a45, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T10:16:38,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for aee80e883adf83e92edd542a79d1ff7e in 506ms, sequenceid=112, compaction requested=false 2024-11-09T10:16:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-09T10:16:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-09T10:16:38,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T10:16:38,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 967 msec 2024-11-09T10:16:38,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 977 msec 2024-11-09T10:16:38,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T10:16:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:38,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/41bfbd84989a4423afaac5bdd26f665d is 50, key is test_row_0/A:col10/1731147398162/Put/seqid=0 2024-11-09T10:16:38,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147458501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742062_1238 (size=14391) 2024-11-09T10:16:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147458503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147458504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147458504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147458505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T10:16:38,560 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:38,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-09T10:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T10:16:38,563 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:38,564 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:38,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147458609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147458619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147458619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147458619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147458619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T10:16:38,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T10:16:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:38,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:38,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147458819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147458829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147458829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147458829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147458832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:38,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T10:16:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:38,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T10:16:38,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/41bfbd84989a4423afaac5bdd26f665d 2024-11-09T10:16:38,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/aeb05bf5075f4cd09fbb6116c6b00b22 is 50, key is test_row_0/B:col10/1731147398162/Put/seqid=0 2024-11-09T10:16:38,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742063_1239 (size=12051) 2024-11-09T10:16:38,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/aeb05bf5075f4cd09fbb6116c6b00b22 2024-11-09T10:16:38,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/1e68df2b81334eefae274a0cc68026e2 is 50, key is test_row_0/C:col10/1731147398162/Put/seqid=0 2024-11-09T10:16:38,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742064_1240 (size=12051) 2024-11-09T10:16:39,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T10:16:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147459132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147459139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147459139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147459139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147459139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T10:16:39,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T10:16:39,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T10:16:39,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:39,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:39,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/1e68df2b81334eefae274a0cc68026e2 2024-11-09T10:16:39,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/41bfbd84989a4423afaac5bdd26f665d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/41bfbd84989a4423afaac5bdd26f665d 2024-11-09T10:16:39,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/41bfbd84989a4423afaac5bdd26f665d, entries=200, sequenceid=132, filesize=14.1 K 2024-11-09T10:16:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/aeb05bf5075f4cd09fbb6116c6b00b22 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/aeb05bf5075f4cd09fbb6116c6b00b22 2024-11-09T10:16:39,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/aeb05bf5075f4cd09fbb6116c6b00b22, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T10:16:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/1e68df2b81334eefae274a0cc68026e2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1e68df2b81334eefae274a0cc68026e2 2024-11-09T10:16:39,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1e68df2b81334eefae274a0cc68026e2, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T10:16:39,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for aee80e883adf83e92edd542a79d1ff7e in 941ms, sequenceid=132, compaction requested=true 2024-11-09T10:16:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:39,422 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:16:39,423 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:39,424 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:39,424 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:39,424 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:39,424 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:39,424 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,424 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fa79d967df684d278f7424fa1fbedc90, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4a761012085746d9aa64d62be44e0d66, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/41bfbd84989a4423afaac5bdd26f665d] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=37.7 K 2024-11-09T10:16:39,424 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,424 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/6aa5907b37ff4dfdb6529b8e2f385a34, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/ffd671b032554b1aa5f82b5528343787, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/aeb05bf5075f4cd09fbb6116c6b00b22] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.4 K 2024-11-09T10:16:39,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa5907b37ff4dfdb6529b8e2f385a34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731147396641 2024-11-09T10:16:39,425 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa79d967df684d278f7424fa1fbedc90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731147396641 2024-11-09T10:16:39,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting ffd671b032554b1aa5f82b5528343787, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147396980 2024-11-09T10:16:39,425 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a761012085746d9aa64d62be44e0d66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147396980 2024-11-09T10:16:39,426 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb05bf5075f4cd09fbb6116c6b00b22, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147398158 2024-11-09T10:16:39,426 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41bfbd84989a4423afaac5bdd26f665d, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147398158 2024-11-09T10:16:39,433 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:39,433 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/95ac9aa25c544b6680f6a99a5204665c is 50, key is test_row_0/A:col10/1731147398162/Put/seqid=0 2024-11-09T10:16:39,433 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:39,434 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/579263fee1b14e63900428e7d7af7831 is 50, key is test_row_0/B:col10/1731147398162/Put/seqid=0 2024-11-09T10:16:39,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742065_1241 (size=12359) 2024-11-09T10:16:39,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742066_1242 (size=12359) 2024-11-09T10:16:39,470 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/579263fee1b14e63900428e7d7af7831 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/579263fee1b14e63900428e7d7af7831 2024-11-09T10:16:39,476 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 579263fee1b14e63900428e7d7af7831(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:39,476 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:39,476 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147399422; duration=0sec 2024-11-09T10:16:39,476 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:39,476 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:39,476 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:39,478 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:39,478 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:39,478 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,478 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/f321c71582ad412aab2e9a2722845b97, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5b831444325a4f5a98e77d1311690a45, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1e68df2b81334eefae274a0cc68026e2] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.4 K 2024-11-09T10:16:39,478 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting f321c71582ad412aab2e9a2722845b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731147396641 2024-11-09T10:16:39,479 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b831444325a4f5a98e77d1311690a45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147396980 2024-11-09T10:16:39,479 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e68df2b81334eefae274a0cc68026e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147398158 2024-11-09T10:16:39,490 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:39,491 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/0caaee02381e4d4c97201665c6b85fb3 is 50, key is test_row_0/C:col10/1731147398162/Put/seqid=0 2024-11-09T10:16:39,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T10:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:39,511 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:39,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:39,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d1f88a9135e844ad96ffc8b4e98e29cf is 50, key is test_row_0/A:col10/1731147398502/Put/seqid=0 2024-11-09T10:16:39,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742067_1243 (size=12359) 2024-11-09T10:16:39,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742068_1244 (size=12151) 2024-11-09T10:16:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:39,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147459672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147459670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147459672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147459679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147459679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T10:16:39,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147459789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147459789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147459789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147459790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147459799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:39,859 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/95ac9aa25c544b6680f6a99a5204665c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/95ac9aa25c544b6680f6a99a5204665c 2024-11-09T10:16:39,867 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into 95ac9aa25c544b6680f6a99a5204665c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:39,867 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:39,867 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147399422; duration=0sec 2024-11-09T10:16:39,867 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:39,867 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:39,938 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d1f88a9135e844ad96ffc8b4e98e29cf 2024-11-09T10:16:39,948 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/0caaee02381e4d4c97201665c6b85fb3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0caaee02381e4d4c97201665c6b85fb3 2024-11-09T10:16:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/e5d025da72014ea6baaf168e77b51683 is 50, key is test_row_0/B:col10/1731147398502/Put/seqid=0 2024-11-09T10:16:39,958 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into 0caaee02381e4d4c97201665c6b85fb3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:39,959 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:39,959 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147399422; duration=0sec 2024-11-09T10:16:39,959 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:39,959 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:39,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742069_1245 (size=12151) 2024-11-09T10:16:40,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147459999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147459999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147459999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147460000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147460009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147460309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147460309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147460309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147460309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147460319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/e5d025da72014ea6baaf168e77b51683 2024-11-09T10:16:40,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5c0881e103824ab1ae437ab81e050ae1 is 50, key is test_row_0/C:col10/1731147398502/Put/seqid=0 2024-11-09T10:16:40,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742070_1246 (size=12151) 2024-11-09T10:16:40,475 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5c0881e103824ab1ae437ab81e050ae1 2024-11-09T10:16:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/d1f88a9135e844ad96ffc8b4e98e29cf as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d1f88a9135e844ad96ffc8b4e98e29cf 2024-11-09T10:16:40,489 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d1f88a9135e844ad96ffc8b4e98e29cf, entries=150, sequenceid=148, filesize=11.9 K 2024-11-09T10:16:40,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/e5d025da72014ea6baaf168e77b51683 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/e5d025da72014ea6baaf168e77b51683 2024-11-09T10:16:40,497 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/e5d025da72014ea6baaf168e77b51683, entries=150, sequenceid=148, filesize=11.9 K 2024-11-09T10:16:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5c0881e103824ab1ae437ab81e050ae1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5c0881e103824ab1ae437ab81e050ae1 2024-11-09T10:16:40,505 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5c0881e103824ab1ae437ab81e050ae1, entries=150, sequenceid=148, filesize=11.9 K 2024-11-09T10:16:40,506 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for aee80e883adf83e92edd542a79d1ff7e in 995ms, sequenceid=148, compaction requested=false 2024-11-09T10:16:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-09T10:16:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-09T10:16:40,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-09T10:16:40,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9450 sec 2024-11-09T10:16:40,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9520 sec 2024-11-09T10:16:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T10:16:40,710 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:40,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-09T10:16:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T10:16:40,715 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:40,716 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:40,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T10:16:40,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:16:40,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:40,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:40,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:40,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:40,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:40,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:40,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/656f2ffcf8404824802fbf05eee2b8ea is 50, key is test_row_0/A:col10/1731147400824/Put/seqid=0 2024-11-09T10:16:40,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742071_1247 (size=12151) 2024-11-09T10:16:40,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/656f2ffcf8404824802fbf05eee2b8ea 2024-11-09T10:16:40,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147460850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147460854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147460861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:40,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147460852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147460869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/1e113d49b8834f4eb031cdeecc1c0ca1 is 50, key is test_row_0/B:col10/1731147400824/Put/seqid=0 2024-11-09T10:16:40,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742072_1248 (size=12151) 2024-11-09T10:16:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147460979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147460980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147460980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147460980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:40,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147460989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:41,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:41,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T10:16:41,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147461189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147461199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147461199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147461200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147461210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T10:16:41,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/1e113d49b8834f4eb031cdeecc1c0ca1 2024-11-09T10:16:41,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5576f7c78b894b9282d68649255d390b is 50, key is test_row_0/C:col10/1731147400824/Put/seqid=0 2024-11-09T10:16:41,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742073_1249 (size=12151) 2024-11-09T10:16:41,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147461499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147461519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147461519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147461519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147461519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:41,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:41,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:41,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5576f7c78b894b9282d68649255d390b 2024-11-09T10:16:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T10:16:41,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/656f2ffcf8404824802fbf05eee2b8ea as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/656f2ffcf8404824802fbf05eee2b8ea 2024-11-09T10:16:41,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/656f2ffcf8404824802fbf05eee2b8ea, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T10:16:41,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/1e113d49b8834f4eb031cdeecc1c0ca1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/1e113d49b8834f4eb031cdeecc1c0ca1 2024-11-09T10:16:41,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/1e113d49b8834f4eb031cdeecc1c0ca1, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T10:16:41,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5576f7c78b894b9282d68649255d390b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5576f7c78b894b9282d68649255d390b 2024-11-09T10:16:41,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5576f7c78b894b9282d68649255d390b, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T10:16:41,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for aee80e883adf83e92edd542a79d1ff7e in 1056ms, sequenceid=169, compaction requested=true 2024-11-09T10:16:41,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:41,884 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:41,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:41,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:41,885 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:41,886 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:41,886 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:41,886 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,886 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/95ac9aa25c544b6680f6a99a5204665c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d1f88a9135e844ad96ffc8b4e98e29cf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/656f2ffcf8404824802fbf05eee2b8ea] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.8 K 2024-11-09T10:16:41,886 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:41,886 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:41,886 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,886 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/579263fee1b14e63900428e7d7af7831, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/e5d025da72014ea6baaf168e77b51683, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/1e113d49b8834f4eb031cdeecc1c0ca1] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.8 K 2024-11-09T10:16:41,887 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ac9aa25c544b6680f6a99a5204665c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147398158 2024-11-09T10:16:41,887 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 579263fee1b14e63900428e7d7af7831, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147398158 2024-11-09T10:16:41,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:41,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:41,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:41,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:41,888 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f88a9135e844ad96ffc8b4e98e29cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731147398502 2024-11-09T10:16:41,888 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d025da72014ea6baaf168e77b51683, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731147398502 2024-11-09T10:16:41,889 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656f2ffcf8404824802fbf05eee2b8ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731147399670 2024-11-09T10:16:41,889 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e113d49b8834f4eb031cdeecc1c0ca1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731147399670 2024-11-09T10:16:41,906 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:41,906 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/d3a293bed82e4b79b79159c608403ccb is 50, key is test_row_0/B:col10/1731147400824/Put/seqid=0 2024-11-09T10:16:41,911 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:41,912 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/e7f431f5f83745ebb20ac83b5563bb6d is 50, key is test_row_0/A:col10/1731147400824/Put/seqid=0 2024-11-09T10:16:41,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T10:16:41,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,960 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:16:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:41,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742074_1250 (size=12561) 2024-11-09T10:16:41,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/bb4dcf74e57f40a5b240cb4342e3f4a9 is 50, key is test_row_0/A:col10/1731147400850/Put/seqid=0 2024-11-09T10:16:41,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742075_1251 (size=12561) 2024-11-09T10:16:41,973 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/d3a293bed82e4b79b79159c608403ccb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d3a293bed82e4b79b79159c608403ccb 2024-11-09T10:16:41,981 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into d3a293bed82e4b79b79159c608403ccb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:41,981 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:41,981 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147401884; duration=0sec 2024-11-09T10:16:41,981 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:41,981 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:41,981 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:41,982 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:41,982 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:41,982 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:41,982 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0caaee02381e4d4c97201665c6b85fb3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5c0881e103824ab1ae437ab81e050ae1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5576f7c78b894b9282d68649255d390b] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=35.8 K 2024-11-09T10:16:41,983 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0caaee02381e4d4c97201665c6b85fb3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147398158 2024-11-09T10:16:41,983 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c0881e103824ab1ae437ab81e050ae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731147398502 2024-11-09T10:16:41,984 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5576f7c78b894b9282d68649255d390b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731147399670 2024-11-09T10:16:41,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742076_1252 (size=12151) 2024-11-09T10:16:41,997 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:41,997 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/862928159e7747caa84fba73fdb8094d is 50, key is test_row_0/C:col10/1731147400824/Put/seqid=0 2024-11-09T10:16:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:42,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742077_1253 (size=12561) 2024-11-09T10:16:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147462050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147462052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147462052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147462054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147462054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147462169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147462170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147462170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147462170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147462170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,381 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/e7f431f5f83745ebb20ac83b5563bb6d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/e7f431f5f83745ebb20ac83b5563bb6d 2024-11-09T10:16:42,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147462380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147462380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,389 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/bb4dcf74e57f40a5b240cb4342e3f4a9 2024-11-09T10:16:42,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,393 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into e7f431f5f83745ebb20ac83b5563bb6d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:42,393 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:42,393 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147401884; duration=0sec 2024-11-09T10:16:42,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147462384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,393 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:42,393 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:42,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147462384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147462385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/40f3349f3cf443ee8581f6675477e184 is 50, key is test_row_0/B:col10/1731147400850/Put/seqid=0 2024-11-09T10:16:42,437 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/862928159e7747caa84fba73fdb8094d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/862928159e7747caa84fba73fdb8094d 2024-11-09T10:16:42,443 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into 862928159e7747caa84fba73fdb8094d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:42,443 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:42,444 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147401888; duration=0sec 2024-11-09T10:16:42,444 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:42,444 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:42,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742078_1254 (size=12151) 2024-11-09T10:16:42,447 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/40f3349f3cf443ee8581f6675477e184 2024-11-09T10:16:42,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/88a71df6c95947e5ba1a7525468c5870 is 50, key is test_row_0/C:col10/1731147400850/Put/seqid=0 2024-11-09T10:16:42,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742079_1255 (size=12151) 2024-11-09T10:16:42,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147462699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147462699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147462699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147462699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147462699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T10:16:42,902 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/88a71df6c95947e5ba1a7525468c5870 2024-11-09T10:16:42,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/bb4dcf74e57f40a5b240cb4342e3f4a9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/bb4dcf74e57f40a5b240cb4342e3f4a9 2024-11-09T10:16:42,913 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/bb4dcf74e57f40a5b240cb4342e3f4a9, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T10:16:42,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/40f3349f3cf443ee8581f6675477e184 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/40f3349f3cf443ee8581f6675477e184 2024-11-09T10:16:42,924 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/40f3349f3cf443ee8581f6675477e184, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T10:16:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/88a71df6c95947e5ba1a7525468c5870 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/88a71df6c95947e5ba1a7525468c5870 2024-11-09T10:16:42,943 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/88a71df6c95947e5ba1a7525468c5870, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T10:16:42,950 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aee80e883adf83e92edd542a79d1ff7e in 990ms, sequenceid=182, compaction requested=false 2024-11-09T10:16:42,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:42,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-09T10:16:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-09T10:16:42,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T10:16:42,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2360 sec 2024-11-09T10:16:42,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2440 sec 2024-11-09T10:16:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:43,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-11-09T10:16:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147402050, max timestamp=1731147403214Num uniques 859; 2024-11-09T10:16:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:16:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147402050, max timestamp=1731147403214Num uniques 859; 2024-11-09T10:16:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:16:43,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147402050, max timestamp=1731147403215Num uniques 859; 2024-11-09T10:16:43,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:43,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/929764e4c18e41ca8886a3cd8af25f40 is 50, key is test_row_0/A:col10/1731147403216/Put/seqid=0 2024-11-09T10:16:43,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742080_1256 (size=16931) 2024-11-09T10:16:43,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147403260, max timestamp=1731147403277Num uniques 859; 2024-11-09T10:16:43,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147403260, max timestamp=1731147403276Num uniques 859; 2024-11-09T10:16:43,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147403260, max timestamp=1731147403276Num uniques 859; 2024-11-09T10:16:43,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:43,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:43,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:43,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147403317, max timestamp=1731147403346Num uniques 1018; 2024-11-09T10:16:43,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T10:16:43,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147403317, max timestamp=1731147403346Num uniques 1018; 2024-11-09T10:16:43,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T10:16:43,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731147403317, max timestamp=1731147403346Num uniques 1018; 2024-11-09T10:16:43,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-09T10:16:43,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147463392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731147403383, max timestamp=1731147403408Num uniques 1027; 2024-11-09T10:16:43,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731147403383, max timestamp=1731147403408Num uniques 1009; 2024-11-09T10:16:43,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-09T10:16:43,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731147403383, max timestamp=1731147403408Num uniques 1027; 2024-11-09T10:16:43,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-09T10:16:43,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-09T10:16:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147463391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147463475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147463476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147463491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147463519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147463519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147463599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147463599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147463609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/929764e4c18e41ca8886a3cd8af25f40 2024-11-09T10:16:43,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/d8cd18d667c84ee6a1e9d68a7429fc39 is 50, key is test_row_0/B:col10/1731147403216/Put/seqid=0 2024-11-09T10:16:43,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742081_1257 (size=12151) 2024-11-09T10:16:43,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147463729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147463740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147463809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147463818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:43,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147463829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147464049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147464058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147464119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147464129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/d8cd18d667c84ee6a1e9d68a7429fc39 2024-11-09T10:16:44,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/8d924e0acfa4414b9c1a1659e2874c38 is 50, key is test_row_0/C:col10/1731147403216/Put/seqid=0 2024-11-09T10:16:44,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147464138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742082_1258 (size=12151) 2024-11-09T10:16:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147464559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/8d924e0acfa4414b9c1a1659e2874c38 2024-11-09T10:16:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147464570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/929764e4c18e41ca8886a3cd8af25f40 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/929764e4c18e41ca8886a3cd8af25f40 2024-11-09T10:16:44,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/929764e4c18e41ca8886a3cd8af25f40, entries=250, sequenceid=208, filesize=16.5 K 2024-11-09T10:16:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/d8cd18d667c84ee6a1e9d68a7429fc39 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d8cd18d667c84ee6a1e9d68a7429fc39 2024-11-09T10:16:44,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d8cd18d667c84ee6a1e9d68a7429fc39, entries=150, sequenceid=208, filesize=11.9 K 2024-11-09T10:16:44,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/8d924e0acfa4414b9c1a1659e2874c38 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/8d924e0acfa4414b9c1a1659e2874c38 2024-11-09T10:16:44,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/8d924e0acfa4414b9c1a1659e2874c38, entries=150, sequenceid=208, filesize=11.9 K 2024-11-09T10:16:44,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~44.73 KB/45800, heapSize ~116.19 KB/118976, currentSize=147.60 KB/151140 for aee80e883adf83e92edd542a79d1ff7e in 1412ms, sequenceid=208, compaction requested=true 2024-11-09T10:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:44,628 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:44,628 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:44,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:44,629 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:44,629 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:44,630 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:44,630 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:44,630 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:44,630 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/e7f431f5f83745ebb20ac83b5563bb6d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/bb4dcf74e57f40a5b240cb4342e3f4a9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/929764e4c18e41ca8886a3cd8af25f40] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=40.7 K 2024-11-09T10:16:44,630 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:44,630 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d3a293bed82e4b79b79159c608403ccb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/40f3349f3cf443ee8581f6675477e184, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d8cd18d667c84ee6a1e9d68a7429fc39] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.0 K 2024-11-09T10:16:44,631 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a293bed82e4b79b79159c608403ccb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731147399670 2024-11-09T10:16:44,631 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f431f5f83745ebb20ac83b5563bb6d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731147399670 2024-11-09T10:16:44,632 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb4dcf74e57f40a5b240cb4342e3f4a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147400846 2024-11-09T10:16:44,632 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f3349f3cf443ee8581f6675477e184, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147400846 2024-11-09T10:16:44,637 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting d8cd18d667c84ee6a1e9d68a7429fc39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731147402052 2024-11-09T10:16:44,637 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929764e4c18e41ca8886a3cd8af25f40, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731147402052 2024-11-09T10:16:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:44,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-11-09T10:16:44,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731147403449, max timestamp=1731147404642Num uniques 1136; 2024-11-09T10:16:44,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:44,648 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:44,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731147403449, max timestamp=1731147404642Num uniques 1136; 2024-11-09T10:16:44,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-09T10:16:44,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731147403449, max timestamp=1731147404642Num uniques 1136; 2024-11-09T10:16:44,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-09T10:16:44,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-09T10:16:44,649 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/3d088b24fa8b42a6a748ff96d893e075 is 50, key is test_row_0/A:col10/1731147403216/Put/seqid=0 2024-11-09T10:16:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:44,662 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:44,663 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/4a36beb3182e421f93d113049b4959f9 is 50, key is test_row_0/B:col10/1731147403216/Put/seqid=0 2024-11-09T10:16:44,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/14e81894be81479dbca92b8157d22d00 is 50, key is test_row_0/A:col10/1731147403468/Put/seqid=0 2024-11-09T10:16:44,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742083_1259 (size=12663) 2024-11-09T10:16:44,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742084_1260 (size=12663) 2024-11-09T10:16:44,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147404684, max timestamp=1731147404719Num uniques 859; 2024-11-09T10:16:44,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:44,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147404684, max timestamp=1731147404719Num uniques 859; 2024-11-09T10:16:44,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:44,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147404684, max timestamp=1731147404719Num uniques 859; 2024-11-09T10:16:44,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:44,735 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/4a36beb3182e421f93d113049b4959f9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4a36beb3182e421f93d113049b4959f9 2024-11-09T10:16:44,739 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/3d088b24fa8b42a6a748ff96d893e075 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3d088b24fa8b42a6a748ff96d893e075 2024-11-09T10:16:44,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742085_1261 (size=12301) 2024-11-09T10:16:44,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/14e81894be81479dbca92b8157d22d00 2024-11-09T10:16:44,749 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 4a36beb3182e421f93d113049b4959f9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:44,749 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:44,749 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147404628; duration=0sec 2024-11-09T10:16:44,749 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:44,749 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:44,750 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:44,751 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into 3d088b24fa8b42a6a748ff96d893e075(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:44,751 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:44,751 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147404628; duration=0sec 2024-11-09T10:16:44,751 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:44,751 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:44,752 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:44,752 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:44,753 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:44,753 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/862928159e7747caa84fba73fdb8094d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/88a71df6c95947e5ba1a7525468c5870, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/8d924e0acfa4414b9c1a1659e2874c38] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.0 K 2024-11-09T10:16:44,753 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 862928159e7747caa84fba73fdb8094d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731147399670 2024-11-09T10:16:44,754 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a71df6c95947e5ba1a7525468c5870, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147400846 2024-11-09T10:16:44,754 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d924e0acfa4414b9c1a1659e2874c38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731147402052 2024-11-09T10:16:44,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/9561f301483b40baa4aa41927b43ab63 is 50, key is test_row_0/B:col10/1731147403468/Put/seqid=0 2024-11-09T10:16:44,771 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:44,771 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/e0124cfa0bda401e88e1a74f7638f291 is 50, key is test_row_0/C:col10/1731147403216/Put/seqid=0 2024-11-09T10:16:44,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742086_1262 (size=12301) 2024-11-09T10:16:44,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/9561f301483b40baa4aa41927b43ab63 2024-11-09T10:16:44,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731147404764, max timestamp=1731147404792Num uniques 1018; 2024-11-09T10:16:44,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731147404764, max timestamp=1731147404792Num uniques 1018; 2024-11-09T10:16:44,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-09T10:16:44,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-09T10:16:44,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731147404764, max timestamp=1731147404792Num uniques 1018; 2024-11-09T10:16:44,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-09T10:16:44,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/53e4a4da8355488ba389c32b79c5d2af is 50, key is test_row_0/C:col10/1731147403468/Put/seqid=0 2024-11-09T10:16:44,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742087_1263 (size=12663) 2024-11-09T10:16:44,812 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/e0124cfa0bda401e88e1a74f7638f291 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e0124cfa0bda401e88e1a74f7638f291 2024-11-09T10:16:44,820 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into e0124cfa0bda401e88e1a74f7638f291(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:44,820 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:44,820 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147404628; duration=0sec 2024-11-09T10:16:44,820 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:44,820 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:44,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742088_1264 (size=12301) 2024-11-09T10:16:44,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/53e4a4da8355488ba389c32b79c5d2af 2024-11-09T10:16:44,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/14e81894be81479dbca92b8157d22d00 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/14e81894be81479dbca92b8157d22d00 2024-11-09T10:16:44,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/14e81894be81479dbca92b8157d22d00, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T10:16:44,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/9561f301483b40baa4aa41927b43ab63 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9561f301483b40baa4aa41927b43ab63 2024-11-09T10:16:44,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731147404814, max timestamp=1731147404842Num uniques 1127; 2024-11-09T10:16:44,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731147404814, max timestamp=1731147404842Num uniques 1127; 2024-11-09T10:16:44,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-09T10:16:44,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-09T10:16:44,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9561f301483b40baa4aa41927b43ab63, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T10:16:44,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/53e4a4da8355488ba389c32b79c5d2af as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/53e4a4da8355488ba389c32b79c5d2af 2024-11-09T10:16:44,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731147404814, max timestamp=1731147404842Num uniques 1127; 2024-11-09T10:16:44,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-09T10:16:44,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/53e4a4da8355488ba389c32b79c5d2af, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T10:16:44,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~63.66 KB/65184, currentSize=67.09 KB/68700 for aee80e883adf83e92edd542a79d1ff7e in 216ms, sequenceid=280, compaction requested=false 2024-11-09T10:16:44,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T10:16:44,869 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:44,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-09T10:16:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T10:16:44,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:44,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:44,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:44,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=87.22 KB heapSize=243.84 KB 2024-11-09T10:16:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:44,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/51802dd41c75420a9584f364cfe1d96e is 50, key is test_row_0/A:col10/1731147404866/Put/seqid=0 2024-11-09T10:16:44,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742089_1265 (size=12301) 2024-11-09T10:16:44,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/51802dd41c75420a9584f364cfe1d96e 2024-11-09T10:16:44,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/b1425af5673f4ba1a2a344d28214b33a is 50, key is test_row_0/B:col10/1731147404866/Put/seqid=0 2024-11-09T10:16:44,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742090_1266 (size=12301) 2024-11-09T10:16:44,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147464936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147464941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147464943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:44,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:44,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T10:16:45,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T10:16:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:45,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147465049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147465058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147465058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T10:16:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T10:16:45,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147465259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147465269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147465269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T10:16:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/b1425af5673f4ba1a2a344d28214b33a 2024-11-09T10:16:45,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/cd216a52097f48daa642a7442a5c37b1 is 50, key is test_row_0/C:col10/1731147404866/Put/seqid=0 2024-11-09T10:16:45,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742091_1267 (size=12301) 2024-11-09T10:16:45,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T10:16:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T10:16:45,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147465569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147465569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,578 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in aee80e883adf83e92edd542a79d1ff7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147465579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,591 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in aee80e883adf83e92edd542a79d1ff7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:45,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147465589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147465590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:45,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T10:16:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T10:16:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:45,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/cd216a52097f48daa642a7442a5c37b1 2024-11-09T10:16:45,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/51802dd41c75420a9584f364cfe1d96e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/51802dd41c75420a9584f364cfe1d96e 2024-11-09T10:16:45,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/51802dd41c75420a9584f364cfe1d96e, entries=150, sequenceid=347, filesize=12.0 K 2024-11-09T10:16:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/b1425af5673f4ba1a2a344d28214b33a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b1425af5673f4ba1a2a344d28214b33a 2024-11-09T10:16:45,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b1425af5673f4ba1a2a344d28214b33a, entries=150, sequenceid=347, filesize=12.0 K 2024-11-09T10:16:45,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/cd216a52097f48daa642a7442a5c37b1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/cd216a52097f48daa642a7442a5c37b1 2024-11-09T10:16:45,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/cd216a52097f48daa642a7442a5c37b1, entries=150, sequenceid=347, filesize=12.0 K 2024-11-09T10:16:45,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~243.80 KB/249648, currentSize=93.93 KB/96180 for aee80e883adf83e92edd542a79d1ff7e in 930ms, sequenceid=347, compaction requested=true 2024-11-09T10:16:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:45,836 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:45,836 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:45,837 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:45,837 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:45,837 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,837 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4a36beb3182e421f93d113049b4959f9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9561f301483b40baa4aa41927b43ab63, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b1425af5673f4ba1a2a344d28214b33a] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.4 K 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:45,838 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a36beb3182e421f93d113049b4959f9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731147402052 2024-11-09T10:16:45,838 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3d088b24fa8b42a6a748ff96d893e075, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/14e81894be81479dbca92b8157d22d00, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/51802dd41c75420a9584f364cfe1d96e] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.4 K 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9561f301483b40baa4aa41927b43ab63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731147403449 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d088b24fa8b42a6a748ff96d893e075, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731147402052 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e81894be81479dbca92b8157d22d00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731147403449 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b1425af5673f4ba1a2a344d28214b33a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147404860 2024-11-09T10:16:45,838 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51802dd41c75420a9584f364cfe1d96e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147404860 2024-11-09T10:16:45,847 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:45,847 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/77f78ee9b28442cc83fb3c3d4a246368 is 50, key is test_row_0/B:col10/1731147404866/Put/seqid=0 2024-11-09T10:16:45,850 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:45,851 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/35a04e33ac7043098369da984f8357a0 is 50, key is test_row_0/A:col10/1731147404866/Put/seqid=0 2024-11-09T10:16:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742092_1268 (size=12915) 2024-11-09T10:16:45,865 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/77f78ee9b28442cc83fb3c3d4a246368 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/77f78ee9b28442cc83fb3c3d4a246368 2024-11-09T10:16:45,872 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 77f78ee9b28442cc83fb3c3d4a246368(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:45,872 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:45,872 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147405836; duration=0sec 2024-11-09T10:16:45,872 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:45,872 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:45,872 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:45,873 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:45,874 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:45,874 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,874 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e0124cfa0bda401e88e1a74f7638f291, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/53e4a4da8355488ba389c32b79c5d2af, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/cd216a52097f48daa642a7442a5c37b1] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.4 K 2024-11-09T10:16:45,874 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e0124cfa0bda401e88e1a74f7638f291, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731147402052 2024-11-09T10:16:45,875 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e4a4da8355488ba389c32b79c5d2af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731147403449 2024-11-09T10:16:45,875 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting cd216a52097f48daa642a7442a5c37b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147404860 2024-11-09T10:16:45,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742093_1269 (size=12915) 2024-11-09T10:16:45,885 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/35a04e33ac7043098369da984f8357a0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/35a04e33ac7043098369da984f8357a0 2024-11-09T10:16:45,890 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into 35a04e33ac7043098369da984f8357a0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:45,890 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:45,890 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147405836; duration=0sec 2024-11-09T10:16:45,890 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:45,890 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:45,891 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:45,892 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/bdab1d5be8924273862f01faaa9e9a89 is 50, key is test_row_0/C:col10/1731147404866/Put/seqid=0 2024-11-09T10:16:45,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742094_1270 (size=12915) 2024-11-09T10:16:45,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T10:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:45,952 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T10:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:45,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/4d41dcef74f644a188a5aa53478c7997 is 50, key is test_row_0/A:col10/1731147404934/Put/seqid=0 2024-11-09T10:16:45,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742095_1271 (size=12301) 2024-11-09T10:16:45,969 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/4d41dcef74f644a188a5aa53478c7997 2024-11-09T10:16:45,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/06ff624e095946d5bdf56654cbc7052a is 50, key is test_row_0/B:col10/1731147404934/Put/seqid=0 2024-11-09T10:16:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742096_1272 (size=12301) 2024-11-09T10:16:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T10:16:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:46,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:46,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147466111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147466116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147466117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147466230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147466232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147466232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,300 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/bdab1d5be8924273862f01faaa9e9a89 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/bdab1d5be8924273862f01faaa9e9a89 2024-11-09T10:16:46,304 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into bdab1d5be8924273862f01faaa9e9a89(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:46,305 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:46,305 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147405836; duration=0sec 2024-11-09T10:16:46,305 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:46,305 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:46,380 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/06ff624e095946d5bdf56654cbc7052a 2024-11-09T10:16:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/fc748ab96724489d9ba7c0e6cee017ad is 50, key is test_row_0/C:col10/1731147404934/Put/seqid=0 2024-11-09T10:16:46,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742097_1273 (size=12301) 2024-11-09T10:16:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147466439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147466439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147466439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147466750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147466750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147466751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:46,797 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/fc748ab96724489d9ba7c0e6cee017ad 2024-11-09T10:16:46,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/4d41dcef74f644a188a5aa53478c7997 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4d41dcef74f644a188a5aa53478c7997 2024-11-09T10:16:46,809 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4d41dcef74f644a188a5aa53478c7997, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T10:16:46,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/06ff624e095946d5bdf56654cbc7052a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06ff624e095946d5bdf56654cbc7052a 2024-11-09T10:16:46,825 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06ff624e095946d5bdf56654cbc7052a, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T10:16:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/fc748ab96724489d9ba7c0e6cee017ad as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fc748ab96724489d9ba7c0e6cee017ad 2024-11-09T10:16:46,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fc748ab96724489d9ba7c0e6cee017ad, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T10:16:46,839 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for aee80e883adf83e92edd542a79d1ff7e in 887ms, sequenceid=366, compaction requested=false 2024-11-09T10:16:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-09T10:16:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-09T10:16:46,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T10:16:46,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9700 sec 2024-11-09T10:16:46,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9760 sec 2024-11-09T10:16:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T10:16:47,019 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:47,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-09T10:16:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T10:16:47,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:47,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:47,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T10:16:47,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T10:16:47,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:47,176 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:16:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:47,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/32d47ab39c3545e68f6913737fab3481 is 50, key is test_row_0/A:col10/1731147406110/Put/seqid=0 2024-11-09T10:16:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742098_1274 (size=12301) 2024-11-09T10:16:47,247 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/32d47ab39c3545e68f6913737fab3481 2024-11-09T10:16:47,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/42742df24c0d43caa72e2a651d3ad45f is 50, key is test_row_0/B:col10/1731147406110/Put/seqid=0 2024-11-09T10:16:47,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:47,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742099_1275 (size=12301) 2024-11-09T10:16:47,280 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/42742df24c0d43caa72e2a651d3ad45f 2024-11-09T10:16:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/07939ce6382f4833954b25cda626399a is 50, key is test_row_0/C:col10/1731147406110/Put/seqid=0 2024-11-09T10:16:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742100_1276 (size=12301) 2024-11-09T10:16:47,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147467306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147467306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147467307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T10:16:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147467419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147467419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147467419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147467599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,602 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in aee80e883adf83e92edd542a79d1ff7e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147467609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,611 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in aee80e883adf83e92edd542a79d1ff7e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147467629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147467629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147467630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T10:16:47,705 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/07939ce6382f4833954b25cda626399a 2024-11-09T10:16:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/32d47ab39c3545e68f6913737fab3481 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/32d47ab39c3545e68f6913737fab3481 2024-11-09T10:16:47,713 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/32d47ab39c3545e68f6913737fab3481, entries=150, sequenceid=382, filesize=12.0 K 2024-11-09T10:16:47,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/42742df24c0d43caa72e2a651d3ad45f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/42742df24c0d43caa72e2a651d3ad45f 2024-11-09T10:16:47,718 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/42742df24c0d43caa72e2a651d3ad45f, entries=150, sequenceid=382, filesize=12.0 K 2024-11-09T10:16:47,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/07939ce6382f4833954b25cda626399a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/07939ce6382f4833954b25cda626399a 2024-11-09T10:16:47,723 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/07939ce6382f4833954b25cda626399a, entries=150, sequenceid=382, filesize=12.0 K 2024-11-09T10:16:47,724 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for aee80e883adf83e92edd542a79d1ff7e in 547ms, sequenceid=382, compaction requested=true 2024-11-09T10:16:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-09T10:16:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-09T10:16:47,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-09T10:16:47,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 703 msec 2024-11-09T10:16:47,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 708 msec 2024-11-09T10:16:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:47,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:16:47,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:47,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:47,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:47,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:47,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:47,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:47,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/3fbef8152ece4f95af56ac3f191beda7 is 50, key is test_row_0/A:col10/1731147407306/Put/seqid=0 2024-11-09T10:16:47,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742101_1277 (size=14741) 2024-11-09T10:16:47,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147467958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147467963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:47,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147467965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147468069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147468079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147468081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T10:16:48,159 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:48,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-09T10:16:48,162 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T10:16:48,163 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:48,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T10:16:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147468279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147468289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147468289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T10:16:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/3fbef8152ece4f95af56ac3f191beda7 2024-11-09T10:16:48,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/698a53c6fff04d9cb89158b3f2d14ef4 is 50, key is test_row_0/B:col10/1731147407306/Put/seqid=0 2024-11-09T10:16:48,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742102_1278 (size=12301) 2024-11-09T10:16:48,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/698a53c6fff04d9cb89158b3f2d14ef4 2024-11-09T10:16:48,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/01fe8396c64247e0a169938c23b99de6 is 50, key is test_row_0/C:col10/1731147407306/Put/seqid=0 2024-11-09T10:16:48,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742103_1279 (size=12301) 2024-11-09T10:16:48,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T10:16:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T10:16:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147468589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147468599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147468600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:48,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T10:16:48,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T10:16:48,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T10:16:48,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:48,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:48,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/01fe8396c64247e0a169938c23b99de6 2024-11-09T10:16:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/3fbef8152ece4f95af56ac3f191beda7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3fbef8152ece4f95af56ac3f191beda7 2024-11-09T10:16:48,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3fbef8152ece4f95af56ac3f191beda7, entries=200, sequenceid=401, filesize=14.4 K 2024-11-09T10:16:48,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/698a53c6fff04d9cb89158b3f2d14ef4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/698a53c6fff04d9cb89158b3f2d14ef4 2024-11-09T10:16:48,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/698a53c6fff04d9cb89158b3f2d14ef4, entries=150, sequenceid=401, filesize=12.0 K 2024-11-09T10:16:48,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/01fe8396c64247e0a169938c23b99de6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/01fe8396c64247e0a169938c23b99de6 2024-11-09T10:16:48,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/01fe8396c64247e0a169938c23b99de6, entries=150, sequenceid=401, filesize=12.0 K 2024-11-09T10:16:48,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for aee80e883adf83e92edd542a79d1ff7e in 1003ms, sequenceid=401, compaction requested=true 2024-11-09T10:16:48,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:48,945 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:16:48,947 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:48,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T10:16:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,948 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:16:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:48,950 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:48,950 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:48,950 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,950 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/77f78ee9b28442cc83fb3c3d4a246368, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06ff624e095946d5bdf56654cbc7052a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/42742df24c0d43caa72e2a651d3ad45f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/698a53c6fff04d9cb89158b3f2d14ef4] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=48.7 K 2024-11-09T10:16:48,951 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:48,951 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:48,951 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:48,951 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/35a04e33ac7043098369da984f8357a0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4d41dcef74f644a188a5aa53478c7997, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/32d47ab39c3545e68f6913737fab3481, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3fbef8152ece4f95af56ac3f191beda7] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=51.0 K 2024-11-09T10:16:48,957 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f78ee9b28442cc83fb3c3d4a246368, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147404860 2024-11-09T10:16:48,957 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a04e33ac7043098369da984f8357a0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147404860 2024-11-09T10:16:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/279a1a30aeb341968ac8c47fc99935db is 50, key is test_row_0/A:col10/1731147407962/Put/seqid=0 2024-11-09T10:16:48,965 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d41dcef74f644a188a5aa53478c7997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731147404934 2024-11-09T10:16:48,965 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ff624e095946d5bdf56654cbc7052a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731147404934 2024-11-09T10:16:48,968 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 42742df24c0d43caa72e2a651d3ad45f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731147406110 2024-11-09T10:16:48,969 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d47ab39c3545e68f6913737fab3481, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731147406110 2024-11-09T10:16:48,972 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fbef8152ece4f95af56ac3f191beda7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731147407285 2024-11-09T10:16:48,976 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 698a53c6fff04d9cb89158b3f2d14ef4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731147407305 2024-11-09T10:16:48,999 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:49,006 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/3be3c8139ba0486ca8f4cc8c550f3305 is 50, key is test_row_0/A:col10/1731147407306/Put/seqid=0 2024-11-09T10:16:49,015 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:49,016 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/447f02e615cb401bb2d7ea6c1a730dda is 50, key is test_row_0/B:col10/1731147407306/Put/seqid=0 2024-11-09T10:16:49,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742104_1280 (size=12301) 2024-11-09T10:16:49,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742106_1282 (size=13051) 2024-11-09T10:16:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742105_1281 (size=13051) 2024-11-09T10:16:49,103 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/447f02e615cb401bb2d7ea6c1a730dda as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/447f02e615cb401bb2d7ea6c1a730dda 2024-11-09T10:16:49,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:49,114 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 447f02e615cb401bb2d7ea6c1a730dda(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:49,115 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:49,115 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=12, startTime=1731147408946; duration=0sec 2024-11-09T10:16:49,115 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:49,115 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:49,115 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:16:49,116 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:16:49,118 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:49,118 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:49,118 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/bdab1d5be8924273862f01faaa9e9a89, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fc748ab96724489d9ba7c0e6cee017ad, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/07939ce6382f4833954b25cda626399a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/01fe8396c64247e0a169938c23b99de6] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=48.7 K 2024-11-09T10:16:49,118 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bdab1d5be8924273862f01faaa9e9a89, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147404860 2024-11-09T10:16:49,119 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting fc748ab96724489d9ba7c0e6cee017ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731147404934 2024-11-09T10:16:49,119 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 07939ce6382f4833954b25cda626399a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731147406110 2024-11-09T10:16:49,119 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 01fe8396c64247e0a169938c23b99de6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731147407305 2024-11-09T10:16:49,134 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:49,134 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/e87d7f98a94a447fa15de16f06917f16 is 50, key is test_row_0/C:col10/1731147407306/Put/seqid=0 2024-11-09T10:16:49,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742107_1283 (size=13051) 2024-11-09T10:16:49,157 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/e87d7f98a94a447fa15de16f06917f16 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e87d7f98a94a447fa15de16f06917f16 2024-11-09T10:16:49,180 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into e87d7f98a94a447fa15de16f06917f16(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:49,180 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:49,180 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=12, startTime=1731147408946; duration=0sec 2024-11-09T10:16:49,181 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:49,181 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:49,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147469173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147469174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147469183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T10:16:49,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147469299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147469299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147469300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,437 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/279a1a30aeb341968ac8c47fc99935db 2024-11-09T10:16:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/de23718015924a1a9ffc668353e320c1 is 50, key is test_row_0/B:col10/1731147407962/Put/seqid=0 2024-11-09T10:16:49,498 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/3be3c8139ba0486ca8f4cc8c550f3305 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3be3c8139ba0486ca8f4cc8c550f3305 2024-11-09T10:16:49,506 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into 3be3c8139ba0486ca8f4cc8c550f3305(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:49,506 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:49,506 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=12, startTime=1731147408945; duration=0sec 2024-11-09T10:16:49,506 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:49,506 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:49,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742108_1284 (size=12301) 2024-11-09T10:16:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147469517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147469517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147469520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147469840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147469840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147469850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:49,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:49,920 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/de23718015924a1a9ffc668353e320c1 2024-11-09T10:16:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/1bd04ee1df904e38b8730f498f8ea879 is 50, key is test_row_0/C:col10/1731147407962/Put/seqid=0 2024-11-09T10:16:49,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742109_1285 (size=12301) 2024-11-09T10:16:49,983 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/1bd04ee1df904e38b8730f498f8ea879 2024-11-09T10:16:49,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/279a1a30aeb341968ac8c47fc99935db as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/279a1a30aeb341968ac8c47fc99935db 2024-11-09T10:16:50,001 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/279a1a30aeb341968ac8c47fc99935db, entries=150, sequenceid=414, filesize=12.0 K 2024-11-09T10:16:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/de23718015924a1a9ffc668353e320c1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/de23718015924a1a9ffc668353e320c1 2024-11-09T10:16:50,010 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/de23718015924a1a9ffc668353e320c1, entries=150, sequenceid=414, filesize=12.0 K 2024-11-09T10:16:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/1bd04ee1df904e38b8730f498f8ea879 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1bd04ee1df904e38b8730f498f8ea879 2024-11-09T10:16:50,022 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1bd04ee1df904e38b8730f498f8ea879, entries=150, sequenceid=414, filesize=12.0 K 2024-11-09T10:16:50,023 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for aee80e883adf83e92edd542a79d1ff7e in 1074ms, sequenceid=414, compaction requested=false 2024-11-09T10:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-09T10:16:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-09T10:16:50,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T10:16:50,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-11-09T10:16:50,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8680 sec 2024-11-09T10:16:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T10:16:50,309 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:50,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-09T10:16:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T10:16:50,312 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:50,314 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:50,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:50,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147409164, max timestamp=1731147410361Num uniques 859; 2024-11-09T10:16:50,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147409164, max timestamp=1731147410361Num uniques 859; 2024-11-09T10:16:50,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:50,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:16:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:50,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-09T10:16:50,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:50,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:16:50,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9500002f60df44ddbc5b53194094a4e0 is 50, key is test_row_0/A:col10/1731147410362/Put/seqid=0 2024-11-09T10:16:50,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742110_1286 (size=17181) 2024-11-09T10:16:50,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9500002f60df44ddbc5b53194094a4e0 2024-11-09T10:16:50,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/da623a277b2740d19041519724cf3173 is 50, key is test_row_0/B:col10/1731147410362/Put/seqid=0 2024-11-09T10:16:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T10:16:50,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147470417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147470421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147470423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742111_1287 (size=12301) 2024-11-09T10:16:50,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T10:16:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147470531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147470539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147470539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T10:16:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:50,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T10:16:50,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147470749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147470750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147470750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:50,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T10:16:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:50,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/da623a277b2740d19041519724cf3173 2024-11-09T10:16:50,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/ea2a3122e5de404c9c933943decd25b6 is 50, key is test_row_0/C:col10/1731147410362/Put/seqid=0 2024-11-09T10:16:50,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742112_1288 (size=12301) 2024-11-09T10:16:50,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/ea2a3122e5de404c9c933943decd25b6 2024-11-09T10:16:50,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9500002f60df44ddbc5b53194094a4e0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9500002f60df44ddbc5b53194094a4e0 2024-11-09T10:16:50,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9500002f60df44ddbc5b53194094a4e0, entries=250, sequenceid=439, filesize=16.8 K 2024-11-09T10:16:50,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/da623a277b2740d19041519724cf3173 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/da623a277b2740d19041519724cf3173 2024-11-09T10:16:50,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/da623a277b2740d19041519724cf3173, entries=150, sequenceid=439, filesize=12.0 K 2024-11-09T10:16:50,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/ea2a3122e5de404c9c933943decd25b6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/ea2a3122e5de404c9c933943decd25b6 2024-11-09T10:16:50,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/ea2a3122e5de404c9c933943decd25b6, entries=150, sequenceid=439, filesize=12.0 K 2024-11-09T10:16:50,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for aee80e883adf83e92edd542a79d1ff7e in 568ms, sequenceid=439, compaction requested=true 2024-11-09T10:16:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:50,933 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:50,934 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:50,934 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:50,934 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:50,934 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:50,934 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:50,934 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,935 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,935 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3be3c8139ba0486ca8f4cc8c550f3305, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/279a1a30aeb341968ac8c47fc99935db, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9500002f60df44ddbc5b53194094a4e0] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=41.5 K 2024-11-09T10:16:50,935 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/447f02e615cb401bb2d7ea6c1a730dda, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/de23718015924a1a9ffc668353e320c1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/da623a277b2740d19041519724cf3173] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.8 K 2024-11-09T10:16:50,935 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 447f02e615cb401bb2d7ea6c1a730dda, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731147407305 2024-11-09T10:16:50,936 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3be3c8139ba0486ca8f4cc8c550f3305, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731147407305 2024-11-09T10:16:50,936 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting de23718015924a1a9ffc668353e320c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731147407955 2024-11-09T10:16:50,937 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279a1a30aeb341968ac8c47fc99935db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731147407955 2024-11-09T10:16:50,938 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting da623a277b2740d19041519724cf3173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731147409165 2024-11-09T10:16:50,938 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9500002f60df44ddbc5b53194094a4e0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731147409164 2024-11-09T10:16:50,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T10:16:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T10:16:50,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:50,939 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T10:16:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:50,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/ab033c7951c84d9db1d80ee0a0aee645 is 50, key is test_row_0/A:col10/1731147410421/Put/seqid=0 2024-11-09T10:16:50,954 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:50,955 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/018afa7950eb491987e0539eb7691314 is 50, key is test_row_0/B:col10/1731147410362/Put/seqid=0 2024-11-09T10:16:50,968 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:50,968 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/452ec83a1f3b42a5b76963fc209724ed is 50, key is test_row_0/A:col10/1731147410362/Put/seqid=0 2024-11-09T10:16:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742113_1289 (size=12301) 2024-11-09T10:16:51,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742114_1290 (size=13153) 2024-11-09T10:16:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742115_1291 (size=13153) 2024-11-09T10:16:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:51,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147471105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147471106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147471111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147471219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147471219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147471229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,402 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/ab033c7951c84d9db1d80ee0a0aee645 2024-11-09T10:16:51,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/51ae4b33faad4a7b86ea8689aa642bbd is 50, key is test_row_0/B:col10/1731147410421/Put/seqid=0 2024-11-09T10:16:51,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742116_1292 (size=12301) 2024-11-09T10:16:51,420 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/018afa7950eb491987e0539eb7691314 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/018afa7950eb491987e0539eb7691314 2024-11-09T10:16:51,424 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 018afa7950eb491987e0539eb7691314(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:51,424 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:51,424 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147410933; duration=0sec 2024-11-09T10:16:51,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:51,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:51,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:51,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:51,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:51,425 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:51,426 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e87d7f98a94a447fa15de16f06917f16, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1bd04ee1df904e38b8730f498f8ea879, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/ea2a3122e5de404c9c933943decd25b6] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.8 K 2024-11-09T10:16:51,426 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e87d7f98a94a447fa15de16f06917f16, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731147407305 2024-11-09T10:16:51,426 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd04ee1df904e38b8730f498f8ea879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731147407955 2024-11-09T10:16:51,427 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting ea2a3122e5de404c9c933943decd25b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731147409165 2024-11-09T10:16:51,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147471429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147471429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,432 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:51,433 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/10fd1aeeebf84ac3b5c819b2e29eaeb7 is 50, key is test_row_0/C:col10/1731147410362/Put/seqid=0 2024-11-09T10:16:51,435 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/452ec83a1f3b42a5b76963fc209724ed as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/452ec83a1f3b42a5b76963fc209724ed 2024-11-09T10:16:51,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742117_1293 (size=13153) 2024-11-09T10:16:51,439 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into 452ec83a1f3b42a5b76963fc209724ed(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:51,439 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:51,440 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147410933; duration=0sec 2024-11-09T10:16:51,440 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:51,440 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:51,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147471439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T10:16:51,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55424 deadline: 1731147471618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,621 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in aee80e883adf83e92edd542a79d1ff7e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55432 deadline: 1731147471639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,642 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in aee80e883adf83e92edd542a79d1ff7e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:16:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147471739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147471739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147471749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:51,815 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/51ae4b33faad4a7b86ea8689aa642bbd 2024-11-09T10:16:51,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5f2abf675d8c410780352cf261c4bfbf is 50, key is test_row_0/C:col10/1731147410421/Put/seqid=0 2024-11-09T10:16:51,843 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/10fd1aeeebf84ac3b5c819b2e29eaeb7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/10fd1aeeebf84ac3b5c819b2e29eaeb7 2024-11-09T10:16:51,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742118_1294 (size=12301) 2024-11-09T10:16:51,844 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5f2abf675d8c410780352cf261c4bfbf 2024-11-09T10:16:51,848 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into 10fd1aeeebf84ac3b5c819b2e29eaeb7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:51,848 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:51,848 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147410934; duration=0sec 2024-11-09T10:16:51,849 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:51,849 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/ab033c7951c84d9db1d80ee0a0aee645 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/ab033c7951c84d9db1d80ee0a0aee645 2024-11-09T10:16:51,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/ab033c7951c84d9db1d80ee0a0aee645, entries=150, sequenceid=459, filesize=12.0 K 2024-11-09T10:16:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/51ae4b33faad4a7b86ea8689aa642bbd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/51ae4b33faad4a7b86ea8689aa642bbd 2024-11-09T10:16:51,865 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/51ae4b33faad4a7b86ea8689aa642bbd, entries=150, sequenceid=459, filesize=12.0 K 2024-11-09T10:16:51,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/5f2abf675d8c410780352cf261c4bfbf as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5f2abf675d8c410780352cf261c4bfbf 2024-11-09T10:16:51,881 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5f2abf675d8c410780352cf261c4bfbf, entries=150, sequenceid=459, filesize=12.0 K 2024-11-09T10:16:51,882 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aee80e883adf83e92edd542a79d1ff7e in 943ms, sequenceid=459, compaction requested=false 2024-11-09T10:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-09T10:16:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-09T10:16:51,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T10:16:51,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-09T10:16:51,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5780 sec 2024-11-09T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:52,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:16:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:52,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/7e7212e7adc74bd49c258a6453e104c9 is 50, key is test_row_0/A:col10/1731147412249/Put/seqid=0 2024-11-09T10:16:52,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742119_1295 (size=14741) 2024-11-09T10:16:52,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147472333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147472335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147472336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147472449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147472449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147472449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T10:16:52,459 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:52,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-09T10:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T10:16:52,492 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:16:52,493 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:16:52,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:16:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T10:16:52,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T10:16:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:52,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147472659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147472659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147472659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/7e7212e7adc74bd49c258a6453e104c9 2024-11-09T10:16:52,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/973806ba71594e00becf5808c51e0273 is 50, key is test_row_0/B:col10/1731147412249/Put/seqid=0 2024-11-09T10:16:52,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742120_1296 (size=12301) 2024-11-09T10:16:52,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T10:16:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:52,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T10:16:52,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T10:16:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:52,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:52,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147472969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147472969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147472969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:53,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/973806ba71594e00becf5808c51e0273 2024-11-09T10:16:53,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T10:16:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2a813f26ec1e41afb896d77b7af961de is 50, key is test_row_0/C:col10/1731147412249/Put/seqid=0 2024-11-09T10:16:53,117 INFO [Thread-1029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1029. 2024-11-09T10:16:53,117 DEBUG [Thread-1029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:53,117 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,117 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:53,118 INFO [Thread-1027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1027. 2024-11-09T10:16:53,118 DEBUG [Thread-1027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:53,119 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,119 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T10:16:53,121 INFO [Thread-1023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1023. 2024-11-09T10:16:53,122 DEBUG [Thread-1023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:53,122 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,122 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:53,122 INFO [Thread-1021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1021. 2024-11-09T10:16:53,122 DEBUG [Thread-1021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:53,122 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,122 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:53,123 INFO [Thread-1025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1025. 2024-11-09T10:16:53,123 DEBUG [Thread-1025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:53,123 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,124 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:53,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:53,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742121_1297 (size=12301) 2024-11-09T10:16:53,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T10:16:53,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:53,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T10:16:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:16:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55466 deadline: 1731147473479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55470 deadline: 1731147473479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1731147473479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:16:53,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2a813f26ec1e41afb896d77b7af961de 2024-11-09T10:16:53,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/7e7212e7adc74bd49c258a6453e104c9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/7e7212e7adc74bd49c258a6453e104c9 2024-11-09T10:16:53,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/7e7212e7adc74bd49c258a6453e104c9, entries=200, sequenceid=475, filesize=14.4 K 2024-11-09T10:16:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/973806ba71594e00becf5808c51e0273 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/973806ba71594e00becf5808c51e0273 2024-11-09T10:16:53,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/973806ba71594e00becf5808c51e0273, entries=150, sequenceid=475, filesize=12.0 K 2024-11-09T10:16:53,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/2a813f26ec1e41afb896d77b7af961de as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2a813f26ec1e41afb896d77b7af961de 2024-11-09T10:16:53,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2a813f26ec1e41afb896d77b7af961de, entries=150, sequenceid=475, filesize=12.0 K 2024-11-09T10:16:53,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aee80e883adf83e92edd542a79d1ff7e in 1293ms, sequenceid=475, compaction requested=true 2024-11-09T10:16:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:16:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:53,543 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:16:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:53,543 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:53,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aee80e883adf83e92edd542a79d1ff7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:16:53,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:53,544 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:53,544 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:53,544 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/A is initiating minor compaction (all files) 2024-11-09T10:16:53,544 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/B is initiating minor compaction (all files) 2024-11-09T10:16:53,544 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/B in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,544 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/A in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,545 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/018afa7950eb491987e0539eb7691314, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/51ae4b33faad4a7b86ea8689aa642bbd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/973806ba71594e00becf5808c51e0273] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.9 K 2024-11-09T10:16:53,545 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/452ec83a1f3b42a5b76963fc209724ed, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/ab033c7951c84d9db1d80ee0a0aee645, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/7e7212e7adc74bd49c258a6453e104c9] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=39.3 K 2024-11-09T10:16:53,545 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452ec83a1f3b42a5b76963fc209724ed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731147409165 2024-11-09T10:16:53,545 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 018afa7950eb491987e0539eb7691314, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731147409165 2024-11-09T10:16:53,545 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab033c7951c84d9db1d80ee0a0aee645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731147410408 2024-11-09T10:16:53,545 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ae4b33faad4a7b86ea8689aa642bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731147410408 2024-11-09T10:16:53,545 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e7212e7adc74bd49c258a6453e104c9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731147411101 2024-11-09T10:16:53,545 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 973806ba71594e00becf5808c51e0273, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731147411101 2024-11-09T10:16:53,555 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#B#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:53,556 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/321f51b078264aafb695c7bc5d7c871e is 50, key is test_row_0/B:col10/1731147412249/Put/seqid=0 2024-11-09T10:16:53,557 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:53,557 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/5822065d1101477b9508487097ec5c4d is 50, key is test_row_0/A:col10/1731147412249/Put/seqid=0 2024-11-09T10:16:53,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742122_1298 (size=13255) 2024-11-09T10:16:53,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T10:16:53,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,565 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:16:53,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:16:53,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:53,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:16:53,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:53,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:16:53,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:16:53,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742123_1299 (size=13255) 2024-11-09T10:16:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/1ed598ef54314aa6a6f8bc0edea3baad is 50, key is test_row_0/A:col10/1731147412335/Put/seqid=0 2024-11-09T10:16:53,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742124_1300 (size=12301) 2024-11-09T10:16:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T10:16:53,967 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/321f51b078264aafb695c7bc5d7c871e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/321f51b078264aafb695c7bc5d7c871e 2024-11-09T10:16:53,973 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/5822065d1101477b9508487097ec5c4d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/5822065d1101477b9508487097ec5c4d 2024-11-09T10:16:53,973 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/B of aee80e883adf83e92edd542a79d1ff7e into 321f51b078264aafb695c7bc5d7c871e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:53,973 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:53,973 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/B, priority=13, startTime=1731147413543; duration=0sec 2024-11-09T10:16:53,973 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:16:53,973 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:B 2024-11-09T10:16:53,973 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:16:53,975 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:16:53,975 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): aee80e883adf83e92edd542a79d1ff7e/C is initiating minor compaction (all files) 2024-11-09T10:16:53,975 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aee80e883adf83e92edd542a79d1ff7e/C in TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:53,975 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/10fd1aeeebf84ac3b5c819b2e29eaeb7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5f2abf675d8c410780352cf261c4bfbf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2a813f26ec1e41afb896d77b7af961de] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp, totalSize=36.9 K 2024-11-09T10:16:53,976 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 10fd1aeeebf84ac3b5c819b2e29eaeb7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731147409165 2024-11-09T10:16:53,976 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f2abf675d8c410780352cf261c4bfbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731147410408 2024-11-09T10:16:53,981 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a813f26ec1e41afb896d77b7af961de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731147411101 2024-11-09T10:16:53,984 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/A of aee80e883adf83e92edd542a79d1ff7e into 5822065d1101477b9508487097ec5c4d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:53,984 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:53,984 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/A, priority=13, startTime=1731147413543; duration=0sec 2024-11-09T10:16:53,984 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:53,984 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:A 2024-11-09T10:16:53,984 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/1ed598ef54314aa6a6f8bc0edea3baad 2024-11-09T10:16:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/4e4f1e7b34e6400c9a80d1b04210528b is 50, key is test_row_0/B:col10/1731147412335/Put/seqid=0 2024-11-09T10:16:54,005 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aee80e883adf83e92edd542a79d1ff7e#C#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:16:54,006 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/964e1cc0de344aaeb294a553160914a4 is 50, key is test_row_0/C:col10/1731147412249/Put/seqid=0 2024-11-09T10:16:54,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742125_1301 (size=12301) 2024-11-09T10:16:54,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742126_1302 (size=13255) 2024-11-09T10:16:54,410 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/4e4f1e7b34e6400c9a80d1b04210528b 2024-11-09T10:16:54,419 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/964e1cc0de344aaeb294a553160914a4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/964e1cc0de344aaeb294a553160914a4 2024-11-09T10:16:54,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/3ab5d465734a43959bf26b72133e9f43 is 50, key is test_row_0/C:col10/1731147412335/Put/seqid=0 2024-11-09T10:16:54,425 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aee80e883adf83e92edd542a79d1ff7e/C of aee80e883adf83e92edd542a79d1ff7e into 964e1cc0de344aaeb294a553160914a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:16:54,425 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:54,425 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., storeName=aee80e883adf83e92edd542a79d1ff7e/C, priority=13, startTime=1731147413544; duration=0sec 2024-11-09T10:16:54,426 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:16:54,426 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aee80e883adf83e92edd542a79d1ff7e:C 2024-11-09T10:16:54,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742127_1303 (size=12301) 2024-11-09T10:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:16:54,494 INFO [Thread-1018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1018. 2024-11-09T10:16:54,494 INFO [Thread-1010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1010. 2024-11-09T10:16:54,494 DEBUG [Thread-1018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:54,494 DEBUG [Thread-1010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:54,494 DEBUG [Thread-1018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:54,494 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:54,494 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:54,494 DEBUG [Thread-1018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:54,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:54,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:54,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. as already flushing 2024-11-09T10:16:54,495 INFO [Thread-1016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1016. 2024-11-09T10:16:54,495 DEBUG [Thread-1016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:16:54,495 DEBUG [Thread-1016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:54,496 DEBUG [Thread-1016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:16:54,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T10:16:54,827 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/3ab5d465734a43959bf26b72133e9f43 2024-11-09T10:16:54,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/1ed598ef54314aa6a6f8bc0edea3baad as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1ed598ef54314aa6a6f8bc0edea3baad 2024-11-09T10:16:54,834 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1ed598ef54314aa6a6f8bc0edea3baad, entries=150, sequenceid=494, filesize=12.0 K 2024-11-09T10:16:54,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/4e4f1e7b34e6400c9a80d1b04210528b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4e4f1e7b34e6400c9a80d1b04210528b 2024-11-09T10:16:54,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4e4f1e7b34e6400c9a80d1b04210528b, entries=150, sequenceid=494, filesize=12.0 K 2024-11-09T10:16:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/3ab5d465734a43959bf26b72133e9f43 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/3ab5d465734a43959bf26b72133e9f43 2024-11-09T10:16:54,842 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/3ab5d465734a43959bf26b72133e9f43, entries=150, sequenceid=494, filesize=12.0 K 2024-11-09T10:16:54,843 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for aee80e883adf83e92edd542a79d1ff7e in 1278ms, sequenceid=494, compaction requested=false 2024-11-09T10:16:54,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for aee80e883adf83e92edd542a79d1ff7e: 2024-11-09T10:16:54,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:16:54,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-09T10:16:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-09T10:16:54,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-09T10:16:54,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3510 sec 2024-11-09T10:16:54,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3870 sec 2024-11-09T10:16:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T10:16:56,659 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:16:56,956 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T10:17:01,691 INFO [Thread-1014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1014. 2024-11-09T10:17:01,691 DEBUG [Thread-1014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:01,691 DEBUG [Thread-1014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:01,692 DEBUG [Thread-1014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:01,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:01,710 INFO [Thread-1012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1012. 2024-11-09T10:17:01,710 DEBUG [Thread-1012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:01,710 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:01,710 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:01,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:01,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T10:17:01,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-09T10:17:01,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2521 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2477 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2443 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2531 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2473 2024-11-09T10:17:01,711 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T10:17:01,711 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:17:01,711 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:17:01,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:01,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:01,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:01,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T10:17:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T10:17:01,715 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147421715"}]},"ts":"1731147421715"} 2024-11-09T10:17:01,717 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T10:17:01,717 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T10:17:01,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:17:01,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, UNASSIGN}] 2024-11-09T10:17:01,720 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, UNASSIGN 2024-11-09T10:17:01,721 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aee80e883adf83e92edd542a79d1ff7e, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:01,722 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42133 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=e067b2c2c7b3,38427,1731147328756, table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T10:17:01,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, UNASSIGN because future has completed 2024-11-09T10:17:01,723 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:17:01,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:17:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T10:17:01,875 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:17:01,875 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing aee80e883adf83e92edd542a79d1ff7e, disabling compactions & flushes 2024-11-09T10:17:01,876 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. after waiting 0 ms 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:17:01,876 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing aee80e883adf83e92edd542a79d1ff7e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=A 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=B 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:01,876 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aee80e883adf83e92edd542a79d1ff7e, store=C 2024-11-09T10:17:01,877 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:01,880 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9e561b2a5fcd46f395dad01f31989226 is 50, key is test_row_0/A:col10/1731147421709/Put/seqid=0 2024-11-09T10:17:01,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742128_1304 (size=12301) 2024-11-09T10:17:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T10:17:02,286 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9e561b2a5fcd46f395dad01f31989226 2024-11-09T10:17:02,292 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/848f118eeb724cc683bf2cc7036ca539 is 50, key is test_row_0/B:col10/1731147421709/Put/seqid=0 2024-11-09T10:17:02,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742129_1305 (size=12301) 2024-11-09T10:17:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T10:17:02,696 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/848f118eeb724cc683bf2cc7036ca539 2024-11-09T10:17:02,703 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/0295f70456304e7c875d7de59cf7b7ce is 50, key is test_row_0/C:col10/1731147421709/Put/seqid=0 2024-11-09T10:17:02,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742130_1306 (size=12301) 2024-11-09T10:17:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T10:17:03,110 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/0295f70456304e7c875d7de59cf7b7ce 2024-11-09T10:17:03,115 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/A/9e561b2a5fcd46f395dad01f31989226 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9e561b2a5fcd46f395dad01f31989226 2024-11-09T10:17:03,118 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9e561b2a5fcd46f395dad01f31989226, entries=150, sequenceid=505, filesize=12.0 K 2024-11-09T10:17:03,119 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/B/848f118eeb724cc683bf2cc7036ca539 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/848f118eeb724cc683bf2cc7036ca539 2024-11-09T10:17:03,122 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/848f118eeb724cc683bf2cc7036ca539, entries=150, sequenceid=505, filesize=12.0 K 2024-11-09T10:17:03,123 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/.tmp/C/0295f70456304e7c875d7de59cf7b7ce as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0295f70456304e7c875d7de59cf7b7ce 2024-11-09T10:17:03,126 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0295f70456304e7c875d7de59cf7b7ce, entries=150, sequenceid=505, filesize=12.0 K 2024-11-09T10:17:03,127 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aee80e883adf83e92edd542a79d1ff7e in 1251ms, sequenceid=505, compaction requested=true 2024-11-09T10:17:03,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1e3fe56c3c5f4afe8feea3dcce15ea3b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9243540613dd4ac5bf6ab4d2fd90f174, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fab014b60d02418f9866c505c23a6114, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d170136be5564458a7fb96871ac6c8ac, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d990449e512e4d70ab22e35a15bd8d51, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fa79d967df684d278f7424fa1fbedc90, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/f64804e04b8e43798466463ae918ed2c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4a761012085746d9aa64d62be44e0d66, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/41bfbd84989a4423afaac5bdd26f665d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/95ac9aa25c544b6680f6a99a5204665c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d1f88a9135e844ad96ffc8b4e98e29cf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/e7f431f5f83745ebb20ac83b5563bb6d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/656f2ffcf8404824802fbf05eee2b8ea, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/bb4dcf74e57f40a5b240cb4342e3f4a9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/929764e4c18e41ca8886a3cd8af25f40, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3d088b24fa8b42a6a748ff96d893e075, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/14e81894be81479dbca92b8157d22d00, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/35a04e33ac7043098369da984f8357a0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/51802dd41c75420a9584f364cfe1d96e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4d41dcef74f644a188a5aa53478c7997, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/32d47ab39c3545e68f6913737fab3481, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3fbef8152ece4f95af56ac3f191beda7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3be3c8139ba0486ca8f4cc8c550f3305, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/279a1a30aeb341968ac8c47fc99935db, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9500002f60df44ddbc5b53194094a4e0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/452ec83a1f3b42a5b76963fc209724ed, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/ab033c7951c84d9db1d80ee0a0aee645, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/7e7212e7adc74bd49c258a6453e104c9] to archive 2024-11-09T10:17:03,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:03,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1e3fe56c3c5f4afe8feea3dcce15ea3b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1e3fe56c3c5f4afe8feea3dcce15ea3b 2024-11-09T10:17:03,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9243540613dd4ac5bf6ab4d2fd90f174 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9243540613dd4ac5bf6ab4d2fd90f174 2024-11-09T10:17:03,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fab014b60d02418f9866c505c23a6114 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fab014b60d02418f9866c505c23a6114 2024-11-09T10:17:03,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d170136be5564458a7fb96871ac6c8ac to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d170136be5564458a7fb96871ac6c8ac 2024-11-09T10:17:03,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d990449e512e4d70ab22e35a15bd8d51 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d990449e512e4d70ab22e35a15bd8d51 2024-11-09T10:17:03,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fa79d967df684d278f7424fa1fbedc90 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/fa79d967df684d278f7424fa1fbedc90 2024-11-09T10:17:03,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/f64804e04b8e43798466463ae918ed2c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/f64804e04b8e43798466463ae918ed2c 2024-11-09T10:17:03,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4a761012085746d9aa64d62be44e0d66 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4a761012085746d9aa64d62be44e0d66 2024-11-09T10:17:03,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/41bfbd84989a4423afaac5bdd26f665d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/41bfbd84989a4423afaac5bdd26f665d 2024-11-09T10:17:03,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/95ac9aa25c544b6680f6a99a5204665c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/95ac9aa25c544b6680f6a99a5204665c 2024-11-09T10:17:03,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d1f88a9135e844ad96ffc8b4e98e29cf to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/d1f88a9135e844ad96ffc8b4e98e29cf 2024-11-09T10:17:03,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/e7f431f5f83745ebb20ac83b5563bb6d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/e7f431f5f83745ebb20ac83b5563bb6d 2024-11-09T10:17:03,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/656f2ffcf8404824802fbf05eee2b8ea to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/656f2ffcf8404824802fbf05eee2b8ea 2024-11-09T10:17:03,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/bb4dcf74e57f40a5b240cb4342e3f4a9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/bb4dcf74e57f40a5b240cb4342e3f4a9 2024-11-09T10:17:03,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/929764e4c18e41ca8886a3cd8af25f40 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/929764e4c18e41ca8886a3cd8af25f40 2024-11-09T10:17:03,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3d088b24fa8b42a6a748ff96d893e075 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3d088b24fa8b42a6a748ff96d893e075 2024-11-09T10:17:03,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/14e81894be81479dbca92b8157d22d00 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/14e81894be81479dbca92b8157d22d00 2024-11-09T10:17:03,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/35a04e33ac7043098369da984f8357a0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/35a04e33ac7043098369da984f8357a0 2024-11-09T10:17:03,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/51802dd41c75420a9584f364cfe1d96e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/51802dd41c75420a9584f364cfe1d96e 2024-11-09T10:17:03,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4d41dcef74f644a188a5aa53478c7997 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/4d41dcef74f644a188a5aa53478c7997 2024-11-09T10:17:03,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/32d47ab39c3545e68f6913737fab3481 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/32d47ab39c3545e68f6913737fab3481 2024-11-09T10:17:03,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3fbef8152ece4f95af56ac3f191beda7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3fbef8152ece4f95af56ac3f191beda7 2024-11-09T10:17:03,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3be3c8139ba0486ca8f4cc8c550f3305 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/3be3c8139ba0486ca8f4cc8c550f3305 2024-11-09T10:17:03,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/279a1a30aeb341968ac8c47fc99935db to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/279a1a30aeb341968ac8c47fc99935db 2024-11-09T10:17:03,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9500002f60df44ddbc5b53194094a4e0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9500002f60df44ddbc5b53194094a4e0 2024-11-09T10:17:03,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/452ec83a1f3b42a5b76963fc209724ed to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/452ec83a1f3b42a5b76963fc209724ed 2024-11-09T10:17:03,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/ab033c7951c84d9db1d80ee0a0aee645 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/ab033c7951c84d9db1d80ee0a0aee645 2024-11-09T10:17:03,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/7e7212e7adc74bd49c258a6453e104c9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/7e7212e7adc74bd49c258a6453e104c9 2024-11-09T10:17:03,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9324799642424d84bbfab0a39a6a08df, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/45e58368c00f44128ecd148c75b29fa8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/2dd48cf1a37042b7ad2d2a914c49f488, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/afd33f5e16164f938b0e9bc891a6ebad, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b7fe074c49c84224a461692e401bea97, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/6aa5907b37ff4dfdb6529b8e2f385a34, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06917a62271c444d953e377041d22289, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/ffd671b032554b1aa5f82b5528343787, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/579263fee1b14e63900428e7d7af7831, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/aeb05bf5075f4cd09fbb6116c6b00b22, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/e5d025da72014ea6baaf168e77b51683, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d3a293bed82e4b79b79159c608403ccb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/1e113d49b8834f4eb031cdeecc1c0ca1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/40f3349f3cf443ee8581f6675477e184, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4a36beb3182e421f93d113049b4959f9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d8cd18d667c84ee6a1e9d68a7429fc39, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9561f301483b40baa4aa41927b43ab63, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/77f78ee9b28442cc83fb3c3d4a246368, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b1425af5673f4ba1a2a344d28214b33a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06ff624e095946d5bdf56654cbc7052a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/42742df24c0d43caa72e2a651d3ad45f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/447f02e615cb401bb2d7ea6c1a730dda, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/698a53c6fff04d9cb89158b3f2d14ef4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/de23718015924a1a9ffc668353e320c1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/018afa7950eb491987e0539eb7691314, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/da623a277b2740d19041519724cf3173, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/51ae4b33faad4a7b86ea8689aa642bbd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/973806ba71594e00becf5808c51e0273] to archive 2024-11-09T10:17:03,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:03,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9324799642424d84bbfab0a39a6a08df to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9324799642424d84bbfab0a39a6a08df 2024-11-09T10:17:03,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/45e58368c00f44128ecd148c75b29fa8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/45e58368c00f44128ecd148c75b29fa8 2024-11-09T10:17:03,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/2dd48cf1a37042b7ad2d2a914c49f488 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/2dd48cf1a37042b7ad2d2a914c49f488 2024-11-09T10:17:03,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/afd33f5e16164f938b0e9bc891a6ebad to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/afd33f5e16164f938b0e9bc891a6ebad 2024-11-09T10:17:03,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b7fe074c49c84224a461692e401bea97 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b7fe074c49c84224a461692e401bea97 2024-11-09T10:17:03,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/6aa5907b37ff4dfdb6529b8e2f385a34 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/6aa5907b37ff4dfdb6529b8e2f385a34 2024-11-09T10:17:03,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06917a62271c444d953e377041d22289 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06917a62271c444d953e377041d22289 2024-11-09T10:17:03,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/ffd671b032554b1aa5f82b5528343787 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/ffd671b032554b1aa5f82b5528343787 2024-11-09T10:17:03,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/579263fee1b14e63900428e7d7af7831 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/579263fee1b14e63900428e7d7af7831 2024-11-09T10:17:03,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/aeb05bf5075f4cd09fbb6116c6b00b22 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/aeb05bf5075f4cd09fbb6116c6b00b22 2024-11-09T10:17:03,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/e5d025da72014ea6baaf168e77b51683 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/e5d025da72014ea6baaf168e77b51683 2024-11-09T10:17:03,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d3a293bed82e4b79b79159c608403ccb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d3a293bed82e4b79b79159c608403ccb 2024-11-09T10:17:03,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/1e113d49b8834f4eb031cdeecc1c0ca1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/1e113d49b8834f4eb031cdeecc1c0ca1 2024-11-09T10:17:03,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/40f3349f3cf443ee8581f6675477e184 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/40f3349f3cf443ee8581f6675477e184 2024-11-09T10:17:03,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4a36beb3182e421f93d113049b4959f9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4a36beb3182e421f93d113049b4959f9 2024-11-09T10:17:03,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d8cd18d667c84ee6a1e9d68a7429fc39 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/d8cd18d667c84ee6a1e9d68a7429fc39 2024-11-09T10:17:03,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9561f301483b40baa4aa41927b43ab63 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/9561f301483b40baa4aa41927b43ab63 2024-11-09T10:17:03,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/77f78ee9b28442cc83fb3c3d4a246368 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/77f78ee9b28442cc83fb3c3d4a246368 2024-11-09T10:17:03,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b1425af5673f4ba1a2a344d28214b33a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/b1425af5673f4ba1a2a344d28214b33a 2024-11-09T10:17:03,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06ff624e095946d5bdf56654cbc7052a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/06ff624e095946d5bdf56654cbc7052a 2024-11-09T10:17:03,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/42742df24c0d43caa72e2a651d3ad45f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/42742df24c0d43caa72e2a651d3ad45f 2024-11-09T10:17:03,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/447f02e615cb401bb2d7ea6c1a730dda to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/447f02e615cb401bb2d7ea6c1a730dda 2024-11-09T10:17:03,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/698a53c6fff04d9cb89158b3f2d14ef4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/698a53c6fff04d9cb89158b3f2d14ef4 2024-11-09T10:17:03,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/de23718015924a1a9ffc668353e320c1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/de23718015924a1a9ffc668353e320c1 2024-11-09T10:17:03,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/018afa7950eb491987e0539eb7691314 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/018afa7950eb491987e0539eb7691314 2024-11-09T10:17:03,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/da623a277b2740d19041519724cf3173 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/da623a277b2740d19041519724cf3173 2024-11-09T10:17:03,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/51ae4b33faad4a7b86ea8689aa642bbd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/51ae4b33faad4a7b86ea8689aa642bbd 2024-11-09T10:17:03,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/973806ba71594e00becf5808c51e0273 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/973806ba71594e00becf5808c51e0273 2024-11-09T10:17:03,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fdbebe02f8a242aca23b5847905e5473, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2b80b527cdca4ea69508f1891e34fd38, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5ba6d428b49340f3a51f8ea7bda667f6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/32c224be50ce481090b7558a79b6c9b4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/f321c71582ad412aab2e9a2722845b97, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/71a888680abb4ea0b4bbe525e81e40cd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5b831444325a4f5a98e77d1311690a45, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0caaee02381e4d4c97201665c6b85fb3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1e68df2b81334eefae274a0cc68026e2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5c0881e103824ab1ae437ab81e050ae1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/862928159e7747caa84fba73fdb8094d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5576f7c78b894b9282d68649255d390b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/88a71df6c95947e5ba1a7525468c5870, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e0124cfa0bda401e88e1a74f7638f291, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/8d924e0acfa4414b9c1a1659e2874c38, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/53e4a4da8355488ba389c32b79c5d2af, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/bdab1d5be8924273862f01faaa9e9a89, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/cd216a52097f48daa642a7442a5c37b1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fc748ab96724489d9ba7c0e6cee017ad, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/07939ce6382f4833954b25cda626399a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e87d7f98a94a447fa15de16f06917f16, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/01fe8396c64247e0a169938c23b99de6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1bd04ee1df904e38b8730f498f8ea879, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/10fd1aeeebf84ac3b5c819b2e29eaeb7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/ea2a3122e5de404c9c933943decd25b6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5f2abf675d8c410780352cf261c4bfbf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2a813f26ec1e41afb896d77b7af961de] to archive 2024-11-09T10:17:03,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:03,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fdbebe02f8a242aca23b5847905e5473 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fdbebe02f8a242aca23b5847905e5473 2024-11-09T10:17:03,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2b80b527cdca4ea69508f1891e34fd38 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2b80b527cdca4ea69508f1891e34fd38 2024-11-09T10:17:03,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5ba6d428b49340f3a51f8ea7bda667f6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5ba6d428b49340f3a51f8ea7bda667f6 2024-11-09T10:17:03,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2fc9b8fcb1cf48abbcb2a51a1a88c6df 2024-11-09T10:17:03,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/32c224be50ce481090b7558a79b6c9b4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/32c224be50ce481090b7558a79b6c9b4 2024-11-09T10:17:03,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/f321c71582ad412aab2e9a2722845b97 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/f321c71582ad412aab2e9a2722845b97 2024-11-09T10:17:03,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/71a888680abb4ea0b4bbe525e81e40cd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/71a888680abb4ea0b4bbe525e81e40cd 2024-11-09T10:17:03,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5b831444325a4f5a98e77d1311690a45 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5b831444325a4f5a98e77d1311690a45 2024-11-09T10:17:03,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0caaee02381e4d4c97201665c6b85fb3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0caaee02381e4d4c97201665c6b85fb3 2024-11-09T10:17:03,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1e68df2b81334eefae274a0cc68026e2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1e68df2b81334eefae274a0cc68026e2 2024-11-09T10:17:03,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5c0881e103824ab1ae437ab81e050ae1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5c0881e103824ab1ae437ab81e050ae1 2024-11-09T10:17:03,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/862928159e7747caa84fba73fdb8094d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/862928159e7747caa84fba73fdb8094d 2024-11-09T10:17:03,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5576f7c78b894b9282d68649255d390b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5576f7c78b894b9282d68649255d390b 2024-11-09T10:17:03,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/88a71df6c95947e5ba1a7525468c5870 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/88a71df6c95947e5ba1a7525468c5870 2024-11-09T10:17:03,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e0124cfa0bda401e88e1a74f7638f291 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e0124cfa0bda401e88e1a74f7638f291 2024-11-09T10:17:03,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/8d924e0acfa4414b9c1a1659e2874c38 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/8d924e0acfa4414b9c1a1659e2874c38 2024-11-09T10:17:03,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/53e4a4da8355488ba389c32b79c5d2af to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/53e4a4da8355488ba389c32b79c5d2af 2024-11-09T10:17:03,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/bdab1d5be8924273862f01faaa9e9a89 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/bdab1d5be8924273862f01faaa9e9a89 2024-11-09T10:17:03,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/cd216a52097f48daa642a7442a5c37b1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/cd216a52097f48daa642a7442a5c37b1 2024-11-09T10:17:03,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fc748ab96724489d9ba7c0e6cee017ad to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/fc748ab96724489d9ba7c0e6cee017ad 2024-11-09T10:17:03,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/07939ce6382f4833954b25cda626399a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/07939ce6382f4833954b25cda626399a 2024-11-09T10:17:03,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e87d7f98a94a447fa15de16f06917f16 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/e87d7f98a94a447fa15de16f06917f16 2024-11-09T10:17:03,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/01fe8396c64247e0a169938c23b99de6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/01fe8396c64247e0a169938c23b99de6 2024-11-09T10:17:03,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1bd04ee1df904e38b8730f498f8ea879 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/1bd04ee1df904e38b8730f498f8ea879 2024-11-09T10:17:03,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/10fd1aeeebf84ac3b5c819b2e29eaeb7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/10fd1aeeebf84ac3b5c819b2e29eaeb7 2024-11-09T10:17:03,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/ea2a3122e5de404c9c933943decd25b6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/ea2a3122e5de404c9c933943decd25b6 2024-11-09T10:17:03,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5f2abf675d8c410780352cf261c4bfbf to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/5f2abf675d8c410780352cf261c4bfbf 2024-11-09T10:17:03,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2a813f26ec1e41afb896d77b7af961de to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/2a813f26ec1e41afb896d77b7af961de 2024-11-09T10:17:03,233 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/recovered.edits/508.seqid, newMaxSeqId=508, maxSeqId=1 2024-11-09T10:17:03,234 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. 2024-11-09T10:17:03,234 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for aee80e883adf83e92edd542a79d1ff7e: Waiting for close lock at 1731147421876Running coprocessor pre-close hooks at 1731147421876Disabling compacts and flushes for region at 1731147421876Disabling writes for close at 1731147421876Obtaining lock to block concurrent updates at 1731147421876Preparing flush snapshotting stores in aee80e883adf83e92edd542a79d1ff7e at 1731147421876Finished memstore snapshotting TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731147421877 (+1 ms)Flushing stores of TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e. at 1731147421877Flushing aee80e883adf83e92edd542a79d1ff7e/A: creating writer at 1731147421877Flushing aee80e883adf83e92edd542a79d1ff7e/A: appending metadata at 1731147421880 (+3 ms)Flushing aee80e883adf83e92edd542a79d1ff7e/A: closing flushed file at 1731147421880Flushing aee80e883adf83e92edd542a79d1ff7e/B: creating writer at 1731147422290 (+410 ms)Flushing aee80e883adf83e92edd542a79d1ff7e/B: appending metadata at 1731147422292 (+2 ms)Flushing aee80e883adf83e92edd542a79d1ff7e/B: closing flushed file at 1731147422292Flushing aee80e883adf83e92edd542a79d1ff7e/C: creating writer at 1731147422700 (+408 ms)Flushing aee80e883adf83e92edd542a79d1ff7e/C: appending metadata at 1731147422703 (+3 ms)Flushing aee80e883adf83e92edd542a79d1ff7e/C: closing flushed file at 1731147422703Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7249053a: reopening flushed file at 1731147423114 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e95cb7b: reopening flushed file at 1731147423118 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21ec0b13: reopening flushed file at 1731147423122 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aee80e883adf83e92edd542a79d1ff7e in 1251ms, sequenceid=505, compaction requested=true at 1731147423127 (+5 ms)Writing region close event to WAL at 1731147423230 (+103 ms)Running coprocessor post-close hooks at 1731147423234 (+4 ms)Closed at 1731147423234 2024-11-09T10:17:03,235 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:17:03,236 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aee80e883adf83e92edd542a79d1ff7e, regionState=CLOSED 2024-11-09T10:17:03,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:17:03,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T10:17:03,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure aee80e883adf83e92edd542a79d1ff7e, server=e067b2c2c7b3,38427,1731147328756 in 1.5160 sec 2024-11-09T10:17:03,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-09T10:17:03,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aee80e883adf83e92edd542a79d1ff7e, UNASSIGN in 1.5230 sec 2024-11-09T10:17:03,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-09T10:17:03,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5260 sec 2024-11-09T10:17:03,247 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147423247"}]},"ts":"1731147423247"} 2024-11-09T10:17:03,248 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T10:17:03,248 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T10:17:03,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-09T10:17:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T10:17:03,859 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:03,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T10:17:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:03,861 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T10:17:03,862 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:03,863 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:17:03,865 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/recovered.edits] 2024-11-09T10:17:03,868 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1ed598ef54314aa6a6f8bc0edea3baad to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/1ed598ef54314aa6a6f8bc0edea3baad 2024-11-09T10:17:03,869 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/5822065d1101477b9508487097ec5c4d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/5822065d1101477b9508487097ec5c4d 2024-11-09T10:17:03,870 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9e561b2a5fcd46f395dad01f31989226 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/A/9e561b2a5fcd46f395dad01f31989226 2024-11-09T10:17:03,872 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/321f51b078264aafb695c7bc5d7c871e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/321f51b078264aafb695c7bc5d7c871e 2024-11-09T10:17:03,873 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4e4f1e7b34e6400c9a80d1b04210528b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/4e4f1e7b34e6400c9a80d1b04210528b 2024-11-09T10:17:03,874 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/848f118eeb724cc683bf2cc7036ca539 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/B/848f118eeb724cc683bf2cc7036ca539 2024-11-09T10:17:03,876 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0295f70456304e7c875d7de59cf7b7ce to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/0295f70456304e7c875d7de59cf7b7ce 2024-11-09T10:17:03,877 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/3ab5d465734a43959bf26b72133e9f43 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/3ab5d465734a43959bf26b72133e9f43 2024-11-09T10:17:03,878 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/964e1cc0de344aaeb294a553160914a4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/C/964e1cc0de344aaeb294a553160914a4 2024-11-09T10:17:03,880 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/recovered.edits/508.seqid to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e/recovered.edits/508.seqid 2024-11-09T10:17:03,880 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/aee80e883adf83e92edd542a79d1ff7e 2024-11-09T10:17:03,880 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T10:17:03,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:03,886 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T10:17:03,888 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T10:17:03,889 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:03,889 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T10:17:03,889 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731147423889"}]},"ts":"9223372036854775807"} 2024-11-09T10:17:03,891 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T10:17:03,891 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aee80e883adf83e92edd542a79d1ff7e, NAME => 'TestAcidGuarantees,,1731147390919.aee80e883adf83e92edd542a79d1ff7e.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T10:17:03,891 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T10:17:03,891 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731147423891"}]},"ts":"9223372036854775807"} 2024-11-09T10:17:03,893 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T10:17:03,894 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:03,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-09T10:17:03,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T10:17:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T10:17:03,971 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:03,978 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=515 (was 468) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1172 (was 1165) - AvailableMemoryMB LEAK? - 2024-11-09T10:17:03,988 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=515, ProcessCount=11, AvailableMemoryMB=1171 2024-11-09T10:17:03,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:17:03,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:17:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:03,991 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T10:17:03,992 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:03,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-09T10:17:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-09T10:17:03,992 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T10:17:03,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742131_1307 (size=1061) 2024-11-09T10:17:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-09T10:17:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-09T10:17:04,399 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:17:04,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742132_1308 (size=53) 2024-11-09T10:17:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-09T10:17:04,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:17:04,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6fd4a3e490137b74edc89a46b520f093, disabling compactions & flushes 2024-11-09T10:17:04,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:04,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:04,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. after waiting 0 ms 2024-11-09T10:17:04,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:04,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:04,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6fd4a3e490137b74edc89a46b520f093: Waiting for close lock at 1731147424804Disabling compacts and flushes for region at 1731147424804Disabling writes for close at 1731147424804Writing region close event to WAL at 1731147424804Closed at 1731147424804 2024-11-09T10:17:04,805 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T10:17:04,806 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731147424806"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731147424806"}]},"ts":"1731147424806"} 2024-11-09T10:17:04,807 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T10:17:04,808 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T10:17:04,808 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147424808"}]},"ts":"1731147424808"} 2024-11-09T10:17:04,809 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T10:17:04,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, ASSIGN}] 2024-11-09T10:17:04,811 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, ASSIGN 2024-11-09T10:17:04,811 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, ASSIGN; state=OFFLINE, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=false 2024-11-09T10:17:04,962 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:04,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, ASSIGN because future has completed 2024-11-09T10:17:04,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:17:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-09T10:17:05,196 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:05,196 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:17:05,197 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,198 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:17:05,198 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,198 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,201 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,203 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:05,203 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fd4a3e490137b74edc89a46b520f093 columnFamilyName A 2024-11-09T10:17:05,203 DEBUG [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:05,203 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(327): Store=6fd4a3e490137b74edc89a46b520f093/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:05,204 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,205 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:05,205 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fd4a3e490137b74edc89a46b520f093 columnFamilyName B 2024-11-09T10:17:05,205 DEBUG [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:05,205 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(327): Store=6fd4a3e490137b74edc89a46b520f093/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:05,206 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,207 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:05,207 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fd4a3e490137b74edc89a46b520f093 columnFamilyName C 2024-11-09T10:17:05,207 DEBUG [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:05,207 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(327): Store=6fd4a3e490137b74edc89a46b520f093/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:05,208 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:05,208 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,208 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,209 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,211 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,211 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,211 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:17:05,212 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,214 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:17:05,214 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 6fd4a3e490137b74edc89a46b520f093; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69841823, jitterRate=0.0407242625951767}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:17:05,214 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:05,215 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 6fd4a3e490137b74edc89a46b520f093: Running coprocessor pre-open hook at 1731147425198Writing region info on filesystem at 1731147425198Initializing all the Stores at 1731147425200 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147425200Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147425201 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147425201Cleaning up temporary data from old regions at 1731147425211 (+10 ms)Running coprocessor post-open hooks at 1731147425214 (+3 ms)Region opened successfully at 1731147425215 (+1 ms) 2024-11-09T10:17:05,216 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., pid=89, masterSystemTime=1731147425120 2024-11-09T10:17:05,217 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:05,218 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:05,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=OPEN, openSeqNum=2, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:05,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:17:05,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-09T10:17:05,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 in 253 msec 2024-11-09T10:17:05,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-09T10:17:05,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, ASSIGN in 414 msec 2024-11-09T10:17:05,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T10:17:05,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147425226"}]},"ts":"1731147425226"} 2024-11-09T10:17:05,228 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T10:17:05,229 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T10:17:05,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2390 sec 2024-11-09T10:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-09T10:17:06,140 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:06,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48dcb24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,143 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cfb3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a5bc91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,146 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,152 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T10:17:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e067b2c2c7b3,42133,1731147327850 2024-11-09T10:17:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@e024d 2024-11-09T10:17:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T10:17:06,155 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T10:17:06,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:17:06,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:06,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742133_1309 (size=1097) 2024-11-09T10:17:06,172 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T10:17:06,172 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T10:17:06,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:17:06,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, REOPEN/MOVE}] 2024-11-09T10:17:06,178 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, REOPEN/MOVE 2024-11-09T10:17:06,179 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, REOPEN/MOVE because future has completed 2024-11-09T10:17:06,181 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:17:06,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:17:06,334 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,334 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:17:06,334 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 6fd4a3e490137b74edc89a46b520f093, disabling compactions & flushes 2024-11-09T10:17:06,334 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,334 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,334 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. after waiting 0 ms 2024-11-09T10:17:06,334 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,337 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T10:17:06,338 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,338 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 6fd4a3e490137b74edc89a46b520f093: Waiting for close lock at 1731147426334Running coprocessor pre-close hooks at 1731147426334Disabling compacts and flushes for region at 1731147426334Disabling writes for close at 1731147426334Writing region close event to WAL at 1731147426335 (+1 ms)Running coprocessor post-close hooks at 1731147426338 (+3 ms)Closed at 1731147426338 2024-11-09T10:17:06,338 WARN [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 6fd4a3e490137b74edc89a46b520f093 to self. 2024-11-09T10:17:06,340 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,341 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=CLOSED 2024-11-09T10:17:06,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:17:06,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-09T10:17:06,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 in 161 msec 2024-11-09T10:17:06,345 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, REOPEN/MOVE; state=CLOSED, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=true 2024-11-09T10:17:06,495 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, REOPEN/MOVE because future has completed 2024-11-09T10:17:06,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:17:06,652 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,652 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:17:06,652 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,652 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:17:06,652 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,652 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,653 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,654 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:06,654 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fd4a3e490137b74edc89a46b520f093 columnFamilyName A 2024-11-09T10:17:06,655 DEBUG [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:06,656 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(327): Store=6fd4a3e490137b74edc89a46b520f093/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:06,656 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,657 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:06,657 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fd4a3e490137b74edc89a46b520f093 columnFamilyName B 2024-11-09T10:17:06,657 DEBUG [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:06,658 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(327): Store=6fd4a3e490137b74edc89a46b520f093/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:06,658 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,658 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:06,658 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fd4a3e490137b74edc89a46b520f093 columnFamilyName C 2024-11-09T10:17:06,658 DEBUG [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:06,659 INFO [StoreOpener-6fd4a3e490137b74edc89a46b520f093-1 {}] regionserver.HStore(327): Store=6fd4a3e490137b74edc89a46b520f093/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:06,659 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,659 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,660 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,660 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,661 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,661 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,662 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:17:06,663 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,665 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 6fd4a3e490137b74edc89a46b520f093; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66354796, jitterRate=-0.011236488819122314}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:17:06,665 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,666 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 6fd4a3e490137b74edc89a46b520f093: Running coprocessor pre-open hook at 1731147426652Writing region info on filesystem at 1731147426652Initializing all the Stores at 1731147426653 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147426653Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147426653Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147426653Cleaning up temporary data from old regions at 1731147426661 (+8 ms)Running coprocessor post-open hooks at 1731147426665 (+4 ms)Region opened successfully at 1731147426666 (+1 ms) 2024-11-09T10:17:06,666 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., pid=94, masterSystemTime=1731147426649 2024-11-09T10:17:06,669 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,669 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=OPEN, openSeqNum=5, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:17:06,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-09T10:17:06,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 in 174 msec 2024-11-09T10:17:06,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T10:17:06,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, REOPEN/MOVE in 495 msec 2024-11-09T10:17:06,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-09T10:17:06,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-09T10:17:06,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 518 msec 2024-11-09T10:17:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-09T10:17:06,678 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:06,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5232d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,680 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fc13de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,681 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53db6627, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67a6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d038449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521ee917, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12688331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2edf849c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f3d2d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1456e4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41315383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,692 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a90f3e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32da4b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,697 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1841a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,698 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d16823a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,700 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dac4835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205266ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,703 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@590f8ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77924115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,706 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8e9622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,708 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db2f93e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b20e1d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6485e8af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,711 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5fb686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fbfe4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:06,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:06,714 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21eaaf1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:06,715 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb4b43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:06,716 DEBUG [Thread-1432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,720 DEBUG [Thread-1428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,724 DEBUG [Thread-1437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,730 DEBUG [Thread-1441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,730 DEBUG [Thread-1426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,731 DEBUG [Thread-1434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,731 DEBUG [Thread-1439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,732 DEBUG [Thread-1443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,733 DEBUG [Thread-1443 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,733 DEBUG [Thread-1443 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:06,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:17:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:06,740 DEBUG [Thread-1445 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-09T10:17:06,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,742 DEBUG [Thread-1430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:06,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T10:17:06,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,746 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,747 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:06,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:06,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:17:06,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110935893d711ad943888a37bae768c4e5bc_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147426730/Put/seqid=0 2024-11-09T10:17:06,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147426762, max timestamp=1731147426798Num uniques 859; 2024-11-09T10:17:06,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147426762, max timestamp=1731147426798Num uniques 859; 2024-11-09T10:17:06,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:06,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:06,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147426762, max timestamp=1731147426798Num uniques 859; 2024-11-09T10:17:06,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:06,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742134_1310 (size=12154) 2024-11-09T10:17:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T10:17:06,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147486838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147486838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147486841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147486839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147486843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T10:17:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:06,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:06,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:06,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:06,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147486959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147486960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147486960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147486969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147486969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T10:17:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T10:17:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147487169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147487170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147487170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147487189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147487189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T10:17:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:07,225 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110935893d711ad943888a37bae768c4e5bc_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110935893d711ad943888a37bae768c4e5bc_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:07,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a8a2ab7f349d4dd79f414cec136dfaa8, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:07,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a8a2ab7f349d4dd79f414cec136dfaa8 is 175, key is test_row_0/A:col10/1731147426730/Put/seqid=0 2024-11-09T10:17:07,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742135_1311 (size=30955) 2024-11-09T10:17:07,239 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a8a2ab7f349d4dd79f414cec136dfaa8 2024-11-09T10:17:07,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bf564a5debbb46698b732e3a86871378 is 50, key is test_row_0/B:col10/1731147426730/Put/seqid=0 2024-11-09T10:17:07,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742136_1312 (size=12001) 2024-11-09T10:17:07,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bf564a5debbb46698b732e3a86871378 2024-11-09T10:17:07,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/a4d080c0cc5b4cfca0b7ced2c59457f1 is 50, key is test_row_0/C:col10/1731147426730/Put/seqid=0 2024-11-09T10:17:07,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742137_1313 (size=12001) 2024-11-09T10:17:07,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T10:17:07,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:07,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T10:17:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147487489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147487489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147487489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147487510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147487511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:07,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T10:17:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T10:17:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:07,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/a4d080c0cc5b4cfca0b7ced2c59457f1 2024-11-09T10:17:07,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a8a2ab7f349d4dd79f414cec136dfaa8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a8a2ab7f349d4dd79f414cec136dfaa8 2024-11-09T10:17:07,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a8a2ab7f349d4dd79f414cec136dfaa8, entries=150, sequenceid=14, filesize=30.2 K 2024-11-09T10:17:07,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bf564a5debbb46698b732e3a86871378 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bf564a5debbb46698b732e3a86871378 2024-11-09T10:17:07,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bf564a5debbb46698b732e3a86871378, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T10:17:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/a4d080c0cc5b4cfca0b7ced2c59457f1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a4d080c0cc5b4cfca0b7ced2c59457f1 2024-11-09T10:17:07,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a4d080c0cc5b4cfca0b7ced2c59457f1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T10:17:07,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6fd4a3e490137b74edc89a46b520f093 in 1014ms, sequenceid=14, compaction requested=false 2024-11-09T10:17:07,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:07,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T10:17:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:07,834 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-09T10:17:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:07,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110961aa1109a6f24668bf2bee6accc88f28_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147426834/Put/seqid=0 2024-11-09T10:17:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742138_1314 (size=12154) 2024-11-09T10:17:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:07,859 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110961aa1109a6f24668bf2bee6accc88f28_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961aa1109a6f24668bf2bee6accc88f28_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:07,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/2f69e679f045460ca985c8ad41ee131d, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:07,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/2f69e679f045460ca985c8ad41ee131d is 175, key is test_row_0/A:col10/1731147426834/Put/seqid=0 2024-11-09T10:17:07,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742139_1315 (size=30955) 2024-11-09T10:17:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T10:17:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:08,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:08,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147488022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147488035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147488037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147488037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147488037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147488144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147488160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147488160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147488160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147488172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,275 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/2f69e679f045460ca985c8ad41ee131d 2024-11-09T10:17:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/666e0eb10c6746dca198a8e6a4e9274f is 50, key is test_row_0/B:col10/1731147426834/Put/seqid=0 2024-11-09T10:17:08,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742140_1316 (size=12001) 2024-11-09T10:17:08,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/666e0eb10c6746dca198a8e6a4e9274f 2024-11-09T10:17:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/206240e3aff340469f79eb583f0a9e03 is 50, key is test_row_0/C:col10/1731147426834/Put/seqid=0 2024-11-09T10:17:08,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742141_1317 (size=12001) 2024-11-09T10:17:08,341 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/206240e3aff340469f79eb583f0a9e03 2024-11-09T10:17:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/2f69e679f045460ca985c8ad41ee131d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/2f69e679f045460ca985c8ad41ee131d 2024-11-09T10:17:08,349 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/2f69e679f045460ca985c8ad41ee131d, entries=150, sequenceid=51, filesize=30.2 K 2024-11-09T10:17:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/666e0eb10c6746dca198a8e6a4e9274f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/666e0eb10c6746dca198a8e6a4e9274f 2024-11-09T10:17:08,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/666e0eb10c6746dca198a8e6a4e9274f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T10:17:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/206240e3aff340469f79eb583f0a9e03 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/206240e3aff340469f79eb583f0a9e03 2024-11-09T10:17:08,359 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/206240e3aff340469f79eb583f0a9e03, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T10:17:08,360 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 6fd4a3e490137b74edc89a46b520f093 in 526ms, sequenceid=51, compaction requested=false 2024-11-09T10:17:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-09T10:17:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-09T10:17:08,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-09T10:17:08,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-09T10:17:08,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6270 sec 2024-11-09T10:17:08,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T10:17:08,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:08,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:08,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:08,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:08,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:08,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:08,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094fc94abad1b145d88945db301f5b98e7_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147428364/Put/seqid=0 2024-11-09T10:17:08,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742142_1318 (size=14594) 2024-11-09T10:17:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147428406, max timestamp=1731147428421Num uniques 859; 2024-11-09T10:17:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147428406, max timestamp=1731147428421Num uniques 859; 2024-11-09T10:17:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:08,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147428406, max timestamp=1731147428421Num uniques 859; 2024-11-09T10:17:08,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:08,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147488454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147488455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147488457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147488464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147488468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147488579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147488579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147488579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147488589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147488589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,787 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T10:17:08,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:08,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094fc94abad1b145d88945db301f5b98e7_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fc94abad1b145d88945db301f5b98e7_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:08,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/e5d29d734f234e5984c53a5d6e1089a5, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:08,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/e5d29d734f234e5984c53a5d6e1089a5 is 175, key is test_row_0/A:col10/1731147428364/Put/seqid=0 2024-11-09T10:17:08,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742143_1319 (size=39549) 2024-11-09T10:17:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147488799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147488800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147488800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147488819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147488822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T10:17:08,899 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:08,900 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-09T10:17:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T10:17:08,905 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:08,905 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:08,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T10:17:09,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T10:17:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:09,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147489129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147489129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147489129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147489129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147489148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,208 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/e5d29d734f234e5984c53a5d6e1089a5 2024-11-09T10:17:09,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/5d77a892c02f4dab9465f4eb09677c05 is 50, key is test_row_0/B:col10/1731147428364/Put/seqid=0 2024-11-09T10:17:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742144_1320 (size=12001) 2024-11-09T10:17:09,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T10:17:09,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T10:17:09,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:09,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:09,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:09,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/5d77a892c02f4dab9465f4eb09677c05 2024-11-09T10:17:09,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/dcd08dfcceba49b5a823811c964b4103 is 50, key is test_row_0/C:col10/1731147428364/Put/seqid=0 2024-11-09T10:17:09,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742145_1321 (size=12001) 2024-11-09T10:17:09,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/dcd08dfcceba49b5a823811c964b4103 2024-11-09T10:17:09,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/e5d29d734f234e5984c53a5d6e1089a5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e5d29d734f234e5984c53a5d6e1089a5 2024-11-09T10:17:09,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e5d29d734f234e5984c53a5d6e1089a5, entries=200, sequenceid=63, filesize=38.6 K 2024-11-09T10:17:09,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/5d77a892c02f4dab9465f4eb09677c05 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/5d77a892c02f4dab9465f4eb09677c05 2024-11-09T10:17:09,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/5d77a892c02f4dab9465f4eb09677c05, entries=150, sequenceid=63, filesize=11.7 K 2024-11-09T10:17:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/dcd08dfcceba49b5a823811c964b4103 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dcd08dfcceba49b5a823811c964b4103 2024-11-09T10:17:09,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dcd08dfcceba49b5a823811c964b4103, entries=150, sequenceid=63, filesize=11.7 K 2024-11-09T10:17:09,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 6fd4a3e490137b74edc89a46b520f093 in 909ms, sequenceid=63, compaction requested=true 2024-11-09T10:17:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:09,276 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:09,276 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:09,278 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:09,278 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:09,278 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/B is initiating minor compaction (all files) 2024-11-09T10:17:09,278 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/A is initiating minor compaction (all files) 2024-11-09T10:17:09,278 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/B in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,278 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/A in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,278 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bf564a5debbb46698b732e3a86871378, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/666e0eb10c6746dca198a8e6a4e9274f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/5d77a892c02f4dab9465f4eb09677c05] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.2 K 2024-11-09T10:17:09,278 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a8a2ab7f349d4dd79f414cec136dfaa8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/2f69e679f045460ca985c8ad41ee131d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e5d29d734f234e5984c53a5d6e1089a5] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=99.1 K 2024-11-09T10:17:09,278 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,278 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a8a2ab7f349d4dd79f414cec136dfaa8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/2f69e679f045460ca985c8ad41ee131d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e5d29d734f234e5984c53a5d6e1089a5] 2024-11-09T10:17:09,278 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bf564a5debbb46698b732e3a86871378, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731147426730 2024-11-09T10:17:09,279 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8a2ab7f349d4dd79f414cec136dfaa8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731147426730 2024-11-09T10:17:09,279 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 666e0eb10c6746dca198a8e6a4e9274f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147426834 2024-11-09T10:17:09,280 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f69e679f045460ca985c8ad41ee131d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147426834 2024-11-09T10:17:09,280 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d77a892c02f4dab9465f4eb09677c05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731147428016 2024-11-09T10:17:09,280 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5d29d734f234e5984c53a5d6e1089a5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731147428016 2024-11-09T10:17:09,290 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#B#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:09,291 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/f187ad80acef4a40ade29d75e084e5ce is 50, key is test_row_0/B:col10/1731147428364/Put/seqid=0 2024-11-09T10:17:09,293 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:09,305 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411096072b4f000d642449fb98a7d9298fe21_6fd4a3e490137b74edc89a46b520f093 store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:09,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742146_1322 (size=12104) 2024-11-09T10:17:09,306 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411096072b4f000d642449fb98a7d9298fe21_6fd4a3e490137b74edc89a46b520f093, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:09,307 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096072b4f000d642449fb98a7d9298fe21_6fd4a3e490137b74edc89a46b520f093 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:09,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742147_1323 (size=4469) 2024-11-09T10:17:09,326 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#A#compaction#273 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:09,327 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/fad1a259bf8b4c37b5a5f09a40609b5a is 175, key is test_row_0/A:col10/1731147428364/Put/seqid=0 2024-11-09T10:17:09,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742148_1324 (size=31058) 2024-11-09T10:17:09,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T10:17:09,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,374 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-09T10:17:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:09,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090e5550470f484be591e313b99e9e3a47_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147428457/Put/seqid=0 2024-11-09T10:17:09,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742149_1325 (size=12154) 2024-11-09T10:17:09,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:09,406 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090e5550470f484be591e313b99e9e3a47_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090e5550470f484be591e313b99e9e3a47_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:09,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/40666d3300074249a5935e994a8659ce, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:09,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/40666d3300074249a5935e994a8659ce is 175, key is test_row_0/A:col10/1731147428457/Put/seqid=0 2024-11-09T10:17:09,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742150_1326 (size=30955) 2024-11-09T10:17:09,440 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/40666d3300074249a5935e994a8659ce 2024-11-09T10:17:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b843589417004a92a6e301cdbe926051 is 50, key is test_row_0/B:col10/1731147428457/Put/seqid=0 2024-11-09T10:17:09,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742151_1327 (size=12001) 2024-11-09T10:17:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T10:17:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:09,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:09,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147489675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147489676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147489678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147489679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147489685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,720 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/f187ad80acef4a40ade29d75e084e5ce as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f187ad80acef4a40ade29d75e084e5ce 2024-11-09T10:17:09,730 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/B of 6fd4a3e490137b74edc89a46b520f093 into f187ad80acef4a40ade29d75e084e5ce(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:09,731 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:09,731 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/B, priority=13, startTime=1731147429276; duration=0sec 2024-11-09T10:17:09,731 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:09,731 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:B 2024-11-09T10:17:09,731 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:09,731 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:09,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/C is initiating minor compaction (all files) 2024-11-09T10:17:09,732 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/C in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,732 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a4d080c0cc5b4cfca0b7ced2c59457f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/206240e3aff340469f79eb583f0a9e03, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dcd08dfcceba49b5a823811c964b4103] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.2 K 2024-11-09T10:17:09,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d080c0cc5b4cfca0b7ced2c59457f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731147426730 2024-11-09T10:17:09,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 206240e3aff340469f79eb583f0a9e03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147426834 2024-11-09T10:17:09,732 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting dcd08dfcceba49b5a823811c964b4103, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731147428016 2024-11-09T10:17:09,737 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#C#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:09,737 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/1442b8f52eab450f8db0146f6babdfa6 is 50, key is test_row_0/C:col10/1731147428364/Put/seqid=0 2024-11-09T10:17:09,769 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/fad1a259bf8b4c37b5a5f09a40609b5a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/fad1a259bf8b4c37b5a5f09a40609b5a 2024-11-09T10:17:09,773 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/A of 6fd4a3e490137b74edc89a46b520f093 into fad1a259bf8b4c37b5a5f09a40609b5a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:09,773 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:09,774 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/A, priority=13, startTime=1731147429276; duration=0sec 2024-11-09T10:17:09,774 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:09,774 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:A 2024-11-09T10:17:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742152_1328 (size=12104) 2024-11-09T10:17:09,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147489789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147489789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147489799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147489799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147489809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:09,881 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b843589417004a92a6e301cdbe926051 2024-11-09T10:17:09,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/64c31e9098834665bd20e37ad298d6a0 is 50, key is test_row_0/C:col10/1731147428457/Put/seqid=0 2024-11-09T10:17:09,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742153_1329 (size=12001) 2024-11-09T10:17:09,926 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/64c31e9098834665bd20e37ad298d6a0 2024-11-09T10:17:09,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/40666d3300074249a5935e994a8659ce as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/40666d3300074249a5935e994a8659ce 2024-11-09T10:17:09,945 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/40666d3300074249a5935e994a8659ce, entries=150, sequenceid=97, filesize=30.2 K 2024-11-09T10:17:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b843589417004a92a6e301cdbe926051 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b843589417004a92a6e301cdbe926051 2024-11-09T10:17:09,952 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b843589417004a92a6e301cdbe926051, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T10:17:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/64c31e9098834665bd20e37ad298d6a0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/64c31e9098834665bd20e37ad298d6a0 2024-11-09T10:17:09,958 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/64c31e9098834665bd20e37ad298d6a0, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T10:17:09,959 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 6fd4a3e490137b74edc89a46b520f093 in 585ms, sequenceid=97, compaction requested=false 2024-11-09T10:17:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-09T10:17:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-09T10:17:09,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-09T10:17:09,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0550 sec 2024-11-09T10:17:09,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.0620 sec 2024-11-09T10:17:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:10,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:17:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:10,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110956d69e96dbcb4ac0bef75f0cb55442f3_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147429674/Put/seqid=0 2024-11-09T10:17:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T10:17:10,039 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:10,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-09T10:17:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T10:17:10,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742154_1330 (size=14594) 2024-11-09T10:17:10,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:10,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:10,045 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:10,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:10,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110956d69e96dbcb4ac0bef75f0cb55442f3_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110956d69e96dbcb4ac0bef75f0cb55442f3_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:10,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/b218dc27ed774542bf39e5b4ee3ec586, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:10,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/b218dc27ed774542bf39e5b4ee3ec586 is 175, key is test_row_0/A:col10/1731147429674/Put/seqid=0 2024-11-09T10:17:10,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742155_1331 (size=39549) 2024-11-09T10:17:10,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147490046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147490053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147490063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147490064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147490068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T10:17:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147490168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147490169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,190 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/1442b8f52eab450f8db0146f6babdfa6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1442b8f52eab450f8db0146f6babdfa6 2024-11-09T10:17:10,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147490189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147490189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147490189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,197 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/C of 6fd4a3e490137b74edc89a46b520f093 into 1442b8f52eab450f8db0146f6babdfa6(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:10,197 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:10,197 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/C, priority=13, startTime=1731147429276; duration=0sec 2024-11-09T10:17:10,197 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:10,197 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:C 2024-11-09T10:17:10,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T10:17:10,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:10,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T10:17:10,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:10,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T10:17:10,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147490389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147490392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147490399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147490399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147490399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/b218dc27ed774542bf39e5b4ee3ec586 2024-11-09T10:17:10,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/a66ffed8fdf144879580c6cc64182b16 is 50, key is test_row_0/B:col10/1731147429674/Put/seqid=0 2024-11-09T10:17:10,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742156_1332 (size=12001) 2024-11-09T10:17:10,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T10:17:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T10:17:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T10:17:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147490699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147490709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147490709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147490709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147490718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:10,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T10:17:10,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:10,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:10,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/a66ffed8fdf144879580c6cc64182b16 2024-11-09T10:17:10,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/01c807d4d2ca474187f024fd3b7e69d8 is 50, key is test_row_0/C:col10/1731147429674/Put/seqid=0 2024-11-09T10:17:10,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742157_1333 (size=12001) 2024-11-09T10:17:10,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/01c807d4d2ca474187f024fd3b7e69d8 2024-11-09T10:17:10,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/b218dc27ed774542bf39e5b4ee3ec586 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/b218dc27ed774542bf39e5b4ee3ec586 2024-11-09T10:17:10,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/b218dc27ed774542bf39e5b4ee3ec586, entries=200, sequenceid=112, filesize=38.6 K 2024-11-09T10:17:10,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/a66ffed8fdf144879580c6cc64182b16 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/a66ffed8fdf144879580c6cc64182b16 2024-11-09T10:17:10,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/a66ffed8fdf144879580c6cc64182b16, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T10:17:10,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/01c807d4d2ca474187f024fd3b7e69d8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/01c807d4d2ca474187f024fd3b7e69d8 2024-11-09T10:17:10,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/01c807d4d2ca474187f024fd3b7e69d8, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T10:17:10,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6fd4a3e490137b74edc89a46b520f093 in 900ms, sequenceid=112, compaction requested=true 2024-11-09T10:17:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:10,915 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:10,915 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:10,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:10,916 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:10,916 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:10,916 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/B is initiating minor compaction (all files) 2024-11-09T10:17:10,916 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/A is initiating minor compaction (all files) 2024-11-09T10:17:10,916 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/B in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,916 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/A in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,917 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f187ad80acef4a40ade29d75e084e5ce, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b843589417004a92a6e301cdbe926051, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/a66ffed8fdf144879580c6cc64182b16] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.3 K 2024-11-09T10:17:10,917 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/fad1a259bf8b4c37b5a5f09a40609b5a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/40666d3300074249a5935e994a8659ce, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/b218dc27ed774542bf39e5b4ee3ec586] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=99.2 K 2024-11-09T10:17:10,917 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,917 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/fad1a259bf8b4c37b5a5f09a40609b5a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/40666d3300074249a5935e994a8659ce, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/b218dc27ed774542bf39e5b4ee3ec586] 2024-11-09T10:17:10,917 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting f187ad80acef4a40ade29d75e084e5ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731147428016 2024-11-09T10:17:10,917 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting fad1a259bf8b4c37b5a5f09a40609b5a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731147428016 2024-11-09T10:17:10,918 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b843589417004a92a6e301cdbe926051, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731147428449 2024-11-09T10:17:10,918 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40666d3300074249a5935e994a8659ce, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731147428449 2024-11-09T10:17:10,919 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a66ffed8fdf144879580c6cc64182b16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147429674 2024-11-09T10:17:10,921 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b218dc27ed774542bf39e5b4ee3ec586, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147429674 2024-11-09T10:17:10,940 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:10,941 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#B#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:10,941 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/d6f6cbfc4b5044d68714a7da29f3979a is 50, key is test_row_0/B:col10/1731147429674/Put/seqid=0 2024-11-09T10:17:10,949 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109b550e1f1f9354cecb0a5d75c20bc322d_6fd4a3e490137b74edc89a46b520f093 store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:10,951 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109b550e1f1f9354cecb0a5d75c20bc322d_6fd4a3e490137b74edc89a46b520f093, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:10,951 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b550e1f1f9354cecb0a5d75c20bc322d_6fd4a3e490137b74edc89a46b520f093 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:10,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T10:17:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:10,964 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:17:10,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:10,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:10,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:10,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:10,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:10,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742158_1334 (size=12207) 2024-11-09T10:17:10,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742159_1335 (size=4469) 2024-11-09T10:17:10,987 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#A#compaction#282 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:10,987 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/c1f0816f070a41da9620491afd622ddb is 175, key is test_row_0/A:col10/1731147429674/Put/seqid=0 2024-11-09T10:17:10,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c352ba96005d4d8a83e49fbcf0a09cae_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147430052/Put/seqid=0 2024-11-09T10:17:11,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742161_1337 (size=12204) 2024-11-09T10:17:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,017 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c352ba96005d4d8a83e49fbcf0a09cae_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c352ba96005d4d8a83e49fbcf0a09cae_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:11,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742160_1336 (size=31161) 2024-11-09T10:17:11,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/da7d14e49a1e4909928a04cc0015239c, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:11,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/da7d14e49a1e4909928a04cc0015239c is 175, key is test_row_0/A:col10/1731147430052/Put/seqid=0 2024-11-09T10:17:11,024 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/c1f0816f070a41da9620491afd622ddb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c1f0816f070a41da9620491afd622ddb 2024-11-09T10:17:11,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742162_1338 (size=31005) 2024-11-09T10:17:11,027 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/da7d14e49a1e4909928a04cc0015239c 2024-11-09T10:17:11,032 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/A of 6fd4a3e490137b74edc89a46b520f093 into c1f0816f070a41da9620491afd622ddb(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:11,032 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:11,032 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/A, priority=13, startTime=1731147430915; duration=0sec 2024-11-09T10:17:11,032 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:11,032 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:A 2024-11-09T10:17:11,032 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:11,033 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:11,033 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/C is initiating minor compaction (all files) 2024-11-09T10:17:11,033 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/C in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,033 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1442b8f52eab450f8db0146f6babdfa6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/64c31e9098834665bd20e37ad298d6a0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/01c807d4d2ca474187f024fd3b7e69d8] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.3 K 2024-11-09T10:17:11,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/aa17fcc0ec384f028df4e8196df84077 is 50, key is test_row_0/B:col10/1731147430052/Put/seqid=0 2024-11-09T10:17:11,036 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1442b8f52eab450f8db0146f6babdfa6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731147428016 2024-11-09T10:17:11,037 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c31e9098834665bd20e37ad298d6a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731147428449 2024-11-09T10:17:11,037 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c807d4d2ca474187f024fd3b7e69d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147429674 2024-11-09T10:17:11,070 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:11,070 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/1e5d80170a3f46e7ad26daa91846b9f1 is 50, key is test_row_0/C:col10/1731147429674/Put/seqid=0 2024-11-09T10:17:11,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742163_1339 (size=12051) 2024-11-09T10:17:11,086 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/aa17fcc0ec384f028df4e8196df84077 2024-11-09T10:17:11,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/47c293ef7b504dcd98fcb9838bc7eb66 is 50, key is test_row_0/C:col10/1731147430052/Put/seqid=0 2024-11-09T10:17:11,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742164_1340 (size=12207) 2024-11-09T10:17:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742165_1341 (size=12051) 2024-11-09T10:17:11,128 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/47c293ef7b504dcd98fcb9838bc7eb66 2024-11-09T10:17:11,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/da7d14e49a1e4909928a04cc0015239c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/da7d14e49a1e4909928a04cc0015239c 2024-11-09T10:17:11,139 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/da7d14e49a1e4909928a04cc0015239c, entries=150, sequenceid=132, filesize=30.3 K 2024-11-09T10:17:11,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/aa17fcc0ec384f028df4e8196df84077 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aa17fcc0ec384f028df4e8196df84077 2024-11-09T10:17:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,144 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aa17fcc0ec384f028df4e8196df84077, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T10:17:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/47c293ef7b504dcd98fcb9838bc7eb66 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/47c293ef7b504dcd98fcb9838bc7eb66 2024-11-09T10:17:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,148 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/47c293ef7b504dcd98fcb9838bc7eb66, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T10:17:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,151 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 6fd4a3e490137b74edc89a46b520f093 in 187ms, sequenceid=132, compaction requested=false 2024-11-09T10:17:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-09T10:17:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-09T10:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-09T10:17:11,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1070 sec 2024-11-09T10:17:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1150 sec 2024-11-09T10:17:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T10:17:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,180 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-09T10:17:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,185 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T10:17:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,186 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:11,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T10:17:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:11,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:17:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:11,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e2d5b07d0b0b4430b84303a69a3591c3_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147431317/Put/seqid=0 2024-11-09T10:17:11,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:11,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:11,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742166_1342 (size=22268) 2024-11-09T10:17:11,375 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/d6f6cbfc4b5044d68714a7da29f3979a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/d6f6cbfc4b5044d68714a7da29f3979a 2024-11-09T10:17:11,391 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/B of 6fd4a3e490137b74edc89a46b520f093 into d6f6cbfc4b5044d68714a7da29f3979a(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:11,391 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:11,391 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/B, priority=13, startTime=1731147430915; duration=0sec 2024-11-09T10:17:11,391 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:11,392 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:B 2024-11-09T10:17:11,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731147431361, max timestamp=1731147431416Num uniques 859; 2024-11-09T10:17:11,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-09T10:17:11,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731147431361, max timestamp=1731147431416Num uniques 859; 2024-11-09T10:17:11,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-09T10:17:11,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731147431361, max timestamp=1731147431416Num uniques 859; 2024-11-09T10:17:11,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-09T10:17:11,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T10:17:11,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147491461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147491460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,528 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/1e5d80170a3f46e7ad26daa91846b9f1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1e5d80170a3f46e7ad26daa91846b9f1 2024-11-09T10:17:11,532 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/C of 6fd4a3e490137b74edc89a46b520f093 into 1e5d80170a3f46e7ad26daa91846b9f1(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:11,532 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:11,532 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/C, priority=13, startTime=1731147430915; duration=0sec 2024-11-09T10:17:11,532 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:11,532 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:C 2024-11-09T10:17:11,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147491518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147491521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147491523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147491629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147491629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147491639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:11,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147491649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147491650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:11,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e2d5b07d0b0b4430b84303a69a3591c3_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e2d5b07d0b0b4430b84303a69a3591c3_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:11,762 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/197fcd935ceb45c5b782c03949846484, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:11,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/197fcd935ceb45c5b782c03949846484 is 175, key is test_row_0/A:col10/1731147431317/Put/seqid=0 2024-11-09T10:17:11,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742167_1343 (size=65673) 2024-11-09T10:17:11,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:11,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T10:17:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147491849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147491849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147491849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147491869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147491869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:11,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:11,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147492159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147492170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147492172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/197fcd935ceb45c5b782c03949846484 2024-11-09T10:17:12,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147492189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147492189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/ff17b9bd50c8487c9f4ce5af2f24022a is 50, key is test_row_0/B:col10/1731147431317/Put/seqid=0 2024-11-09T10:17:12,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742168_1344 (size=12151) 2024-11-09T10:17:12,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T10:17:12,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/ff17b9bd50c8487c9f4ce5af2f24022a 2024-11-09T10:17:12,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/36b6d43f839843a3b04becdbe4bfcbad is 50, key is test_row_0/C:col10/1731147431317/Put/seqid=0 2024-11-09T10:17:12,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742169_1345 (size=12151) 2024-11-09T10:17:12,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147492679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147492679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147492692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147492719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147492719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:12,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:13,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:13,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:13,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/36b6d43f839843a3b04becdbe4bfcbad 2024-11-09T10:17:13,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/197fcd935ceb45c5b782c03949846484 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/197fcd935ceb45c5b782c03949846484 2024-11-09T10:17:13,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/197fcd935ceb45c5b782c03949846484, entries=350, sequenceid=143, filesize=64.1 K 2024-11-09T10:17:13,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/ff17b9bd50c8487c9f4ce5af2f24022a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/ff17b9bd50c8487c9f4ce5af2f24022a 2024-11-09T10:17:13,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/ff17b9bd50c8487c9f4ce5af2f24022a, entries=150, sequenceid=143, filesize=11.9 K 2024-11-09T10:17:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/36b6d43f839843a3b04becdbe4bfcbad as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/36b6d43f839843a3b04becdbe4bfcbad 2024-11-09T10:17:13,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/36b6d43f839843a3b04becdbe4bfcbad, entries=150, sequenceid=143, filesize=11.9 K 2024-11-09T10:17:13,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 6fd4a3e490137b74edc89a46b520f093 in 1742ms, sequenceid=143, compaction requested=true 2024-11-09T10:17:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:13,069 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:13,069 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:13,071 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:13,071 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:13,071 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/B is initiating minor compaction (all files) 2024-11-09T10:17:13,071 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/A is initiating minor compaction (all files) 2024-11-09T10:17:13,071 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/A in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:13,071 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/B in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:13,071 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/d6f6cbfc4b5044d68714a7da29f3979a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aa17fcc0ec384f028df4e8196df84077, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/ff17b9bd50c8487c9f4ce5af2f24022a] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.6 K 2024-11-09T10:17:13,071 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c1f0816f070a41da9620491afd622ddb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/da7d14e49a1e4909928a04cc0015239c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/197fcd935ceb45c5b782c03949846484] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=124.8 K 2024-11-09T10:17:13,071 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:13,071 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c1f0816f070a41da9620491afd622ddb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/da7d14e49a1e4909928a04cc0015239c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/197fcd935ceb45c5b782c03949846484] 2024-11-09T10:17:13,071 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f6cbfc4b5044d68714a7da29f3979a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147429674 2024-11-09T10:17:13,072 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f0816f070a41da9620491afd622ddb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147429674 2024-11-09T10:17:13,072 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting aa17fcc0ec384f028df4e8196df84077, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147430043 2024-11-09T10:17:13,072 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7d14e49a1e4909928a04cc0015239c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147430043 2024-11-09T10:17:13,072 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting ff17b9bd50c8487c9f4ce5af2f24022a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731147431313 2024-11-09T10:17:13,073 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 197fcd935ceb45c5b782c03949846484, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731147431261 2024-11-09T10:17:13,082 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#B#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:13,082 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/aaa0339c74f04394a30ddca91d1ce511 is 50, key is test_row_0/B:col10/1731147431317/Put/seqid=0 2024-11-09T10:17:13,089 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:13,093 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110949a6d29768fd4a04a5b63aca4bda3394_6fd4a3e490137b74edc89a46b520f093 store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:13,095 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110949a6d29768fd4a04a5b63aca4bda3394_6fd4a3e490137b74edc89a46b520f093, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:13,095 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110949a6d29768fd4a04a5b63aca4bda3394_6fd4a3e490137b74edc89a46b520f093 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:13,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742170_1346 (size=12459) 2024-11-09T10:17:13,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742171_1347 (size=4469) 2024-11-09T10:17:13,110 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#A#compaction#291 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:13,110 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/be39acacbb4b4d3e96a98fc093d6786f is 175, key is test_row_0/A:col10/1731147431317/Put/seqid=0 2024-11-09T10:17:13,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742172_1348 (size=31413) 2024-11-09T10:17:13,113 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/aaa0339c74f04394a30ddca91d1ce511 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aaa0339c74f04394a30ddca91d1ce511 2024-11-09T10:17:13,117 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/B of 6fd4a3e490137b74edc89a46b520f093 into aaa0339c74f04394a30ddca91d1ce511(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:13,118 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:13,118 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/B, priority=13, startTime=1731147433069; duration=0sec 2024-11-09T10:17:13,118 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:13,118 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:B 2024-11-09T10:17:13,118 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:13,119 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/be39acacbb4b4d3e96a98fc093d6786f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/be39acacbb4b4d3e96a98fc093d6786f 2024-11-09T10:17:13,119 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:13,119 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/C is initiating minor compaction (all files) 2024-11-09T10:17:13,119 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/C in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:13,119 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1e5d80170a3f46e7ad26daa91846b9f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/47c293ef7b504dcd98fcb9838bc7eb66, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/36b6d43f839843a3b04becdbe4bfcbad] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.6 K 2024-11-09T10:17:13,120 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5d80170a3f46e7ad26daa91846b9f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731147429674 2024-11-09T10:17:13,120 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 47c293ef7b504dcd98fcb9838bc7eb66, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731147430043 2024-11-09T10:17:13,123 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b6d43f839843a3b04becdbe4bfcbad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731147431313 2024-11-09T10:17:13,127 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/A of 6fd4a3e490137b74edc89a46b520f093 into be39acacbb4b4d3e96a98fc093d6786f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:13,127 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:13,128 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/A, priority=13, startTime=1731147433069; duration=0sec 2024-11-09T10:17:13,128 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:13,128 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:A 2024-11-09T10:17:13,144 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#C#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:13,145 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/a93148e42f3343ab8a3ea05a41a122c1 is 50, key is test_row_0/C:col10/1731147431317/Put/seqid=0 2024-11-09T10:17:13,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742173_1349 (size=12459) 2024-11-09T10:17:13,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T10:17:13,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:13,188 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=154.31 KB heapSize=421.31 KB 2024-11-09T10:17:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:13,189 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/a93148e42f3343ab8a3ea05a41a122c1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a93148e42f3343ab8a3ea05a41a122c1 2024-11-09T10:17:13,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096fc19732cb93443f894f38e07a9092a4_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147431432/Put/seqid=0 2024-11-09T10:17:13,198 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/C of 6fd4a3e490137b74edc89a46b520f093 into a93148e42f3343ab8a3ea05a41a122c1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:13,198 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:13,198 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/C, priority=13, startTime=1731147433069; duration=0sec 2024-11-09T10:17:13,198 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:13,198 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:C 2024-11-09T10:17:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742174_1350 (size=12304) 2024-11-09T10:17:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T10:17:13,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:13,627 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096fc19732cb93443f894f38e07a9092a4_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096fc19732cb93443f894f38e07a9092a4_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/113e079313c14ca9ba9d73b4dd88b33e, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/113e079313c14ca9ba9d73b4dd88b33e is 175, key is test_row_0/A:col10/1731147431432/Put/seqid=0 2024-11-09T10:17:13,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742175_1351 (size=31105) 2024-11-09T10:17:13,639 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/113e079313c14ca9ba9d73b4dd88b33e 2024-11-09T10:17:13,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bee11505efcd49e8a6f9260a2ccd2af4 is 50, key is test_row_0/B:col10/1731147431432/Put/seqid=0 2024-11-09T10:17:13,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742176_1352 (size=12151) 2024-11-09T10:17:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:13,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:13,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147493717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:13,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147493718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:13,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147493719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:13,726 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147493739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147493739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:13,741 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:13,742 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:13,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147493829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147493839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:14,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147494049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:14,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147494049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:14,074 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bee11505efcd49e8a6f9260a2ccd2af4 2024-11-09T10:17:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/b1f6c73eeb6f45309313c00ba03e5e93 is 50, key is test_row_0/C:col10/1731147431432/Put/seqid=0 2024-11-09T10:17:14,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742177_1353 (size=12151) 2024-11-09T10:17:14,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147494370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147494370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:14,488 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/b1f6c73eeb6f45309313c00ba03e5e93 2024-11-09T10:17:14,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/113e079313c14ca9ba9d73b4dd88b33e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/113e079313c14ca9ba9d73b4dd88b33e 2024-11-09T10:17:14,499 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/113e079313c14ca9ba9d73b4dd88b33e, entries=150, sequenceid=182, filesize=30.4 K 2024-11-09T10:17:14,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bee11505efcd49e8a6f9260a2ccd2af4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bee11505efcd49e8a6f9260a2ccd2af4 2024-11-09T10:17:14,505 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bee11505efcd49e8a6f9260a2ccd2af4, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T10:17:14,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/b1f6c73eeb6f45309313c00ba03e5e93 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1f6c73eeb6f45309313c00ba03e5e93 2024-11-09T10:17:14,515 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1f6c73eeb6f45309313c00ba03e5e93, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T10:17:14,516 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~421.27 KB/431376, currentSize=33.54 KB/34350 for 6fd4a3e490137b74edc89a46b520f093 in 1327ms, sequenceid=182, compaction requested=false 2024-11-09T10:17:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-09T10:17:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-09T10:17:14,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-09T10:17:14,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3310 sec 2024-11-09T10:17:14,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.3400 sec 2024-11-09T10:17:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:14,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:17:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:14,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090b393026a0f5428ca322bca51a8288b4_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147434891/Put/seqid=0 2024-11-09T10:17:14,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742178_1354 (size=17284) 2024-11-09T10:17:14,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:14,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090b393026a0f5428ca322bca51a8288b4_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090b393026a0f5428ca322bca51a8288b4_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:14,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/036aea2d1f904c50854f1a4469f4edd3, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:14,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/036aea2d1f904c50854f1a4469f4edd3 is 175, key is test_row_0/A:col10/1731147434891/Put/seqid=0 2024-11-09T10:17:14,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742179_1355 (size=48389) 2024-11-09T10:17:14,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/036aea2d1f904c50854f1a4469f4edd3 2024-11-09T10:17:14,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/7b8c3e58ad3849c18a272abff4a419fa is 50, key is test_row_0/B:col10/1731147434891/Put/seqid=0 2024-11-09T10:17:14,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742180_1356 (size=12151) 2024-11-09T10:17:14,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/7b8c3e58ad3849c18a272abff4a419fa 2024-11-09T10:17:15,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e1758c2e08db4743a0316776d1669fdc is 50, key is test_row_0/C:col10/1731147434891/Put/seqid=0 2024-11-09T10:17:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742181_1357 (size=12151) 2024-11-09T10:17:15,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147434991, max timestamp=1731147435021Num uniques 859; 2024-11-09T10:17:15,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:15,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147434991, max timestamp=1731147435021Num uniques 859; 2024-11-09T10:17:15,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147434991, max timestamp=1731147435021Num uniques 859; 2024-11-09T10:17:15,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:15,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:15,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e1758c2e08db4743a0316776d1669fdc 2024-11-09T10:17:15,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/036aea2d1f904c50854f1a4469f4edd3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/036aea2d1f904c50854f1a4469f4edd3 2024-11-09T10:17:15,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/036aea2d1f904c50854f1a4469f4edd3, entries=250, sequenceid=193, filesize=47.3 K 2024-11-09T10:17:15,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/7b8c3e58ad3849c18a272abff4a419fa as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7b8c3e58ad3849c18a272abff4a419fa 2024-11-09T10:17:15,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7b8c3e58ad3849c18a272abff4a419fa, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T10:17:15,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e1758c2e08db4743a0316776d1669fdc as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e1758c2e08db4743a0316776d1669fdc 2024-11-09T10:17:15,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e1758c2e08db4743a0316776d1669fdc, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T10:17:15,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=67.09 KB/68700 for 6fd4a3e490137b74edc89a46b520f093 in 157ms, sequenceid=193, compaction requested=true 2024-11-09T10:17:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:15,051 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:15,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:15,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:15,052 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:15,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:15,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:15,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:15,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:15,053 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:15,053 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/A is initiating minor compaction (all files) 2024-11-09T10:17:15,053 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/A in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,053 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/be39acacbb4b4d3e96a98fc093d6786f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/113e079313c14ca9ba9d73b4dd88b33e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/036aea2d1f904c50854f1a4469f4edd3] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=108.3 K 2024-11-09T10:17:15,053 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,053 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/be39acacbb4b4d3e96a98fc093d6786f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/113e079313c14ca9ba9d73b4dd88b33e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/036aea2d1f904c50854f1a4469f4edd3] 2024-11-09T10:17:15,053 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting be39acacbb4b4d3e96a98fc093d6786f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731147431313 2024-11-09T10:17:15,053 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:15,053 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/B is initiating minor compaction (all files) 2024-11-09T10:17:15,053 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/B in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,054 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113e079313c14ca9ba9d73b4dd88b33e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147431432 2024-11-09T10:17:15,054 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aaa0339c74f04394a30ddca91d1ce511, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bee11505efcd49e8a6f9260a2ccd2af4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7b8c3e58ad3849c18a272abff4a419fa] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.9 K 2024-11-09T10:17:15,054 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036aea2d1f904c50854f1a4469f4edd3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147433704 2024-11-09T10:17:15,054 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting aaa0339c74f04394a30ddca91d1ce511, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731147431313 2024-11-09T10:17:15,054 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bee11505efcd49e8a6f9260a2ccd2af4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147431432 2024-11-09T10:17:15,055 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b8c3e58ad3849c18a272abff4a419fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147433712 2024-11-09T10:17:15,059 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:15,060 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109e5d65924aab34db4a28f0bfd3dfb8b75_6fd4a3e490137b74edc89a46b520f093 store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:15,061 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:15,061 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/c400ebd7e72540eda76e33e5d4a30a6f is 50, key is test_row_0/B:col10/1731147434891/Put/seqid=0 2024-11-09T10:17:15,063 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109e5d65924aab34db4a28f0bfd3dfb8b75_6fd4a3e490137b74edc89a46b520f093, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:15,063 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e5d65924aab34db4a28f0bfd3dfb8b75_6fd4a3e490137b74edc89a46b520f093 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:15,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-11-09T10:17:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:15,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742182_1358 (size=12561) 2024-11-09T10:17:15,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c1d664e9862944d8845840c417a1c285_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147435061/Put/seqid=0 2024-11-09T10:17:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742183_1359 (size=4469) 2024-11-09T10:17:15,144 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#A#compaction#300 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:15,144 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/317b0796ab174cadaf976ba4910e82a8 is 175, key is test_row_0/A:col10/1731147434891/Put/seqid=0 2024-11-09T10:17:15,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742184_1360 (size=14794) 2024-11-09T10:17:15,146 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742185_1361 (size=31515) 2024-11-09T10:17:15,151 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c1d664e9862944d8845840c417a1c285_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c1d664e9862944d8845840c417a1c285_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:15,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/cf664d1607454338a015c606a662df4b, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:15,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/cf664d1607454338a015c606a662df4b is 175, key is test_row_0/A:col10/1731147435061/Put/seqid=0 2024-11-09T10:17:15,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742186_1362 (size=39749) 2024-11-09T10:17:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147495200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147495206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147495318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147495331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T10:17:15,359 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:15,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-09T10:17:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T10:17:15,365 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:15,366 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:15,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T10:17:15,486 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/c400ebd7e72540eda76e33e5d4a30a6f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/c400ebd7e72540eda76e33e5d4a30a6f 2024-11-09T10:17:15,490 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/B of 6fd4a3e490137b74edc89a46b520f093 into c400ebd7e72540eda76e33e5d4a30a6f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:15,490 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:15,490 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/B, priority=13, startTime=1731147435052; duration=0sec 2024-11-09T10:17:15,490 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:15,490 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:B 2024-11-09T10:17:15,490 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:15,491 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:15,491 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/C is initiating minor compaction (all files) 2024-11-09T10:17:15,491 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/C in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,491 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a93148e42f3343ab8a3ea05a41a122c1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1f6c73eeb6f45309313c00ba03e5e93, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e1758c2e08db4743a0316776d1669fdc] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=35.9 K 2024-11-09T10:17:15,491 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a93148e42f3343ab8a3ea05a41a122c1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731147431313 2024-11-09T10:17:15,491 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f6c73eeb6f45309313c00ba03e5e93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147431432 2024-11-09T10:17:15,491 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e1758c2e08db4743a0316776d1669fdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147433712 2024-11-09T10:17:15,501 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:15,501 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/dfbd5db7cdad42c3bbedfca3951d613e is 50, key is test_row_0/C:col10/1731147434891/Put/seqid=0 2024-11-09T10:17:15,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742187_1363 (size=12561) 2024-11-09T10:17:15,511 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/dfbd5db7cdad42c3bbedfca3951d613e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dfbd5db7cdad42c3bbedfca3951d613e 2024-11-09T10:17:15,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,525 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/C of 6fd4a3e490137b74edc89a46b520f093 into dfbd5db7cdad42c3bbedfca3951d613e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:15,525 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:15,525 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/C, priority=13, startTime=1731147435052; duration=0sec 2024-11-09T10:17:15,525 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:15,525 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:C 2024-11-09T10:17:15,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147495529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147495550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,565 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/cf664d1607454338a015c606a662df4b 2024-11-09T10:17:15,573 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/317b0796ab174cadaf976ba4910e82a8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/317b0796ab174cadaf976ba4910e82a8 2024-11-09T10:17:15,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bc8f7c2514864628af95d40d3fadd1be is 50, key is test_row_0/B:col10/1731147435061/Put/seqid=0 2024-11-09T10:17:15,581 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/A of 6fd4a3e490137b74edc89a46b520f093 into 317b0796ab174cadaf976ba4910e82a8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:15,581 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:15,581 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/A, priority=13, startTime=1731147435051; duration=0sec 2024-11-09T10:17:15,581 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:15,581 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:A 2024-11-09T10:17:15,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742188_1364 (size=12151) 2024-11-09T10:17:15,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T10:17:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147495739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,748 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147495749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147495749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,761 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:15,761 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147495849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147495869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:15,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:15,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T10:17:15,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bc8f7c2514864628af95d40d3fadd1be 2024-11-09T10:17:16,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/22c0ccf1addf4b9cb77f1eac025cada3 is 50, key is test_row_0/C:col10/1731147435061/Put/seqid=0 2024-11-09T10:17:16,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742189_1365 (size=12151) 2024-11-09T10:17:16,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:16,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:16,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:16,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:16,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:16,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147496369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:16,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147496389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:16,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/22c0ccf1addf4b9cb77f1eac025cada3 2024-11-09T10:17:16,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/cf664d1607454338a015c606a662df4b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/cf664d1607454338a015c606a662df4b 2024-11-09T10:17:16,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/cf664d1607454338a015c606a662df4b, entries=200, sequenceid=221, filesize=38.8 K 2024-11-09T10:17:16,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bc8f7c2514864628af95d40d3fadd1be as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bc8f7c2514864628af95d40d3fadd1be 2024-11-09T10:17:16,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:16,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:16,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bc8f7c2514864628af95d40d3fadd1be, entries=150, sequenceid=221, filesize=11.9 K 2024-11-09T10:17:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/22c0ccf1addf4b9cb77f1eac025cada3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/22c0ccf1addf4b9cb77f1eac025cada3 2024-11-09T10:17:16,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/22c0ccf1addf4b9cb77f1eac025cada3, entries=150, sequenceid=221, filesize=11.9 K 2024-11-09T10:17:16,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=107.34 KB/109920 for 6fd4a3e490137b74edc89a46b520f093 in 1387ms, sequenceid=221, compaction requested=false 2024-11-09T10:17:16,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T10:17:16,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T10:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:16,596 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:16,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b6f12f8b96d1491c8ca2e76c27e41993_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147435204/Put/seqid=0 2024-11-09T10:17:16,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742190_1366 (size=12304) 2024-11-09T10:17:17,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:17,015 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b6f12f8b96d1491c8ca2e76c27e41993_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b6f12f8b96d1491c8ca2e76c27e41993_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/25a39859496f4601af246641d64bf8e1, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:17,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/25a39859496f4601af246641d64bf8e1 is 175, key is test_row_0/A:col10/1731147435204/Put/seqid=0 2024-11-09T10:17:17,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742191_1367 (size=31105) 2024-11-09T10:17:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:17,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:17,421 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/25a39859496f4601af246641d64bf8e1 2024-11-09T10:17:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/638ab9f0661048338da4b36089cbe110 is 50, key is test_row_0/B:col10/1731147435204/Put/seqid=0 2024-11-09T10:17:17,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742192_1368 (size=12151) 2024-11-09T10:17:17,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:17,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147497473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147497473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T10:17:17,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147497599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147497599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147497819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147497819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:17,836 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/638ab9f0661048338da4b36089cbe110 2024-11-09T10:17:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/9a9a293ad29546c49e325983dc563cfb is 50, key is test_row_0/C:col10/1731147435204/Put/seqid=0 2024-11-09T10:17:17,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742193_1369 (size=12151) 2024-11-09T10:17:17,845 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/9a9a293ad29546c49e325983dc563cfb 2024-11-09T10:17:17,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/25a39859496f4601af246641d64bf8e1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/25a39859496f4601af246641d64bf8e1 2024-11-09T10:17:17,866 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/25a39859496f4601af246641d64bf8e1, entries=150, sequenceid=243, filesize=30.4 K 2024-11-09T10:17:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/638ab9f0661048338da4b36089cbe110 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/638ab9f0661048338da4b36089cbe110 2024-11-09T10:17:17,871 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/638ab9f0661048338da4b36089cbe110, entries=150, sequenceid=243, filesize=11.9 K 2024-11-09T10:17:17,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/9a9a293ad29546c49e325983dc563cfb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/9a9a293ad29546c49e325983dc563cfb 2024-11-09T10:17:17,877 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/9a9a293ad29546c49e325983dc563cfb, entries=150, sequenceid=243, filesize=11.9 K 2024-11-09T10:17:17,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 6fd4a3e490137b74edc89a46b520f093 in 1282ms, sequenceid=243, compaction requested=true 2024-11-09T10:17:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-09T10:17:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-09T10:17:17,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-09T10:17:17,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5130 sec 2024-11-09T10:17:17,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.5220 sec 2024-11-09T10:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:18,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:17:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:18,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109660341de5ada48c28314f35b762fef20_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147437440/Put/seqid=0 2024-11-09T10:17:18,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742194_1370 (size=14844) 2024-11-09T10:17:18,155 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:18,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109660341de5ada48c28314f35b762fef20_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109660341de5ada48c28314f35b762fef20_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:18,160 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/027a3f2b5f02427f96a9d6587b55b734, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:18,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/027a3f2b5f02427f96a9d6587b55b734 is 175, key is test_row_0/A:col10/1731147437440/Put/seqid=0 2024-11-09T10:17:18,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742195_1371 (size=39799) 2024-11-09T10:17:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147498204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147498214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147498319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147498339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147498539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147498559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/027a3f2b5f02427f96a9d6587b55b734 2024-11-09T10:17:18,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/07290800757a4793a69741b36e48ab0d is 50, key is test_row_0/B:col10/1731147437440/Put/seqid=0 2024-11-09T10:17:18,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742196_1372 (size=12201) 2024-11-09T10:17:18,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/07290800757a4793a69741b36e48ab0d 2024-11-09T10:17:18,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/0dc53b9d511d47c1a346637b27bf4901 is 50, key is test_row_0/C:col10/1731147437440/Put/seqid=0 2024-11-09T10:17:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742197_1373 (size=12201) 2024-11-09T10:17:18,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/0dc53b9d511d47c1a346637b27bf4901 2024-11-09T10:17:18,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/027a3f2b5f02427f96a9d6587b55b734 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/027a3f2b5f02427f96a9d6587b55b734 2024-11-09T10:17:18,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/027a3f2b5f02427f96a9d6587b55b734, entries=200, sequenceid=258, filesize=38.9 K 2024-11-09T10:17:18,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/07290800757a4793a69741b36e48ab0d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/07290800757a4793a69741b36e48ab0d 2024-11-09T10:17:18,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/07290800757a4793a69741b36e48ab0d, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T10:17:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/0dc53b9d511d47c1a346637b27bf4901 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/0dc53b9d511d47c1a346637b27bf4901 2024-11-09T10:17:18,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/0dc53b9d511d47c1a346637b27bf4901, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T10:17:18,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6fd4a3e490137b74edc89a46b520f093 in 542ms, sequenceid=258, compaction requested=true 2024-11-09T10:17:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:17:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:18,685 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:18,685 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:18,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T10:17:18,691 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49064 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:18,691 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/C is initiating minor compaction (all files) 2024-11-09T10:17:18,691 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/C in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:18,691 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dfbd5db7cdad42c3bbedfca3951d613e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/22c0ccf1addf4b9cb77f1eac025cada3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/9a9a293ad29546c49e325983dc563cfb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/0dc53b9d511d47c1a346637b27bf4901] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=47.9 K 2024-11-09T10:17:18,691 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142168 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:18,691 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/A is initiating minor compaction (all files) 2024-11-09T10:17:18,692 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/A in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:18,692 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/317b0796ab174cadaf976ba4910e82a8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/cf664d1607454338a015c606a662df4b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/25a39859496f4601af246641d64bf8e1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/027a3f2b5f02427f96a9d6587b55b734] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=138.8 K 2024-11-09T10:17:18,692 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:18,692 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/317b0796ab174cadaf976ba4910e82a8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/cf664d1607454338a015c606a662df4b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/25a39859496f4601af246641d64bf8e1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/027a3f2b5f02427f96a9d6587b55b734] 2024-11-09T10:17:18,699 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317b0796ab174cadaf976ba4910e82a8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147433712 2024-11-09T10:17:18,699 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting dfbd5db7cdad42c3bbedfca3951d613e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147433712 2024-11-09T10:17:18,701 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c0ccf1addf4b9cb77f1eac025cada3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147435021 2024-11-09T10:17:18,701 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf664d1607454338a015c606a662df4b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147435021 2024-11-09T10:17:18,702 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a9a293ad29546c49e325983dc563cfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731147435184 2024-11-09T10:17:18,702 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a39859496f4601af246641d64bf8e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731147435184 2024-11-09T10:17:18,702 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027a3f2b5f02427f96a9d6587b55b734, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147437440 2024-11-09T10:17:18,702 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc53b9d511d47c1a346637b27bf4901, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147437440 2024-11-09T10:17:18,717 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#C#compaction#311 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:18,717 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/f316a05179b149c99f03a29566af655d is 50, key is test_row_0/C:col10/1731147437440/Put/seqid=0 2024-11-09T10:17:18,721 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:18,723 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ef34cc30f269465b860f87cc833b69fe_6fd4a3e490137b74edc89a46b520f093 store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:18,726 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ef34cc30f269465b860f87cc833b69fe_6fd4a3e490137b74edc89a46b520f093, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:18,726 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ef34cc30f269465b860f87cc833b69fe_6fd4a3e490137b74edc89a46b520f093 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:18,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742198_1374 (size=12747) 2024-11-09T10:17:18,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742199_1375 (size=4469) 2024-11-09T10:17:18,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T10:17:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:18,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110947bfd70b10ae4942a06446c886fd3aa5_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147438184/Put/seqid=0 2024-11-09T10:17:18,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742200_1376 (size=14994) 2024-11-09T10:17:18,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110947bfd70b10ae4942a06446c886fd3aa5_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110947bfd70b10ae4942a06446c886fd3aa5_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147498950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147498951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:18,954 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/c5cba13e9d274fad8f72ee9575a7d14e, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:18,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/c5cba13e9d274fad8f72ee9575a7d14e is 175, key is test_row_0/A:col10/1731147438184/Put/seqid=0 2024-11-09T10:17:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742201_1377 (size=39949) 2024-11-09T10:17:19,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147499060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147499060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,180 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/f316a05179b149c99f03a29566af655d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/f316a05179b149c99f03a29566af655d 2024-11-09T10:17:19,184 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#A#compaction#312 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:19,185 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/e120bdf91d3740c4971cc2f8c63c6f3b is 175, key is test_row_0/A:col10/1731147437440/Put/seqid=0 2024-11-09T10:17:19,191 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/C of 6fd4a3e490137b74edc89a46b520f093 into f316a05179b149c99f03a29566af655d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:19,191 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:19,191 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/C, priority=12, startTime=1731147438685; duration=0sec 2024-11-09T10:17:19,192 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:19,192 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:C 2024-11-09T10:17:19,192 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:19,204 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49064 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:19,205 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/B is initiating minor compaction (all files) 2024-11-09T10:17:19,205 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/B in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:19,205 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/c400ebd7e72540eda76e33e5d4a30a6f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bc8f7c2514864628af95d40d3fadd1be, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/638ab9f0661048338da4b36089cbe110, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/07290800757a4793a69741b36e48ab0d] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=47.9 K 2024-11-09T10:17:19,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742202_1378 (size=31701) 2024-11-09T10:17:19,205 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c400ebd7e72540eda76e33e5d4a30a6f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147433712 2024-11-09T10:17:19,206 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bc8f7c2514864628af95d40d3fadd1be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147435021 2024-11-09T10:17:19,206 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 638ab9f0661048338da4b36089cbe110, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731147435184 2024-11-09T10:17:19,207 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 07290800757a4793a69741b36e48ab0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147437440 2024-11-09T10:17:19,210 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/e120bdf91d3740c4971cc2f8c63c6f3b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e120bdf91d3740c4971cc2f8c63c6f3b 2024-11-09T10:17:19,219 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#B#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:19,219 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/64419861c7eb41eb8e90e3ba05264308 is 50, key is test_row_0/B:col10/1731147437440/Put/seqid=0 2024-11-09T10:17:19,224 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/A of 6fd4a3e490137b74edc89a46b520f093 into e120bdf91d3740c4971cc2f8c63c6f3b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:19,224 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:19,224 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/A, priority=12, startTime=1731147438685; duration=0sec 2024-11-09T10:17:19,224 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:19,224 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:A 2024-11-09T10:17:19,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742203_1379 (size=12747) 2024-11-09T10:17:19,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147499289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147499289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,364 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/c5cba13e9d274fad8f72ee9575a7d14e 2024-11-09T10:17:19,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b96f8ea63201456685e5db30c55b25b6 is 50, key is test_row_0/B:col10/1731147438184/Put/seqid=0 2024-11-09T10:17:19,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742204_1380 (size=12301) 2024-11-09T10:17:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T10:17:19,519 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:19,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-09T10:17:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T10:17:19,522 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:19,522 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:19,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147499609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147499609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T10:17:19,636 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/64419861c7eb41eb8e90e3ba05264308 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/64419861c7eb41eb8e90e3ba05264308 2024-11-09T10:17:19,642 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/B of 6fd4a3e490137b74edc89a46b520f093 into 64419861c7eb41eb8e90e3ba05264308(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:19,642 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:19,642 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/B, priority=12, startTime=1731147438685; duration=0sec 2024-11-09T10:17:19,642 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:19,642 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:B 2024-11-09T10:17:19,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T10:17:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:19,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:19,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39364 deadline: 1731147499769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,775 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8315 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b96f8ea63201456685e5db30c55b25b6 2024-11-09T10:17:19,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/fcff002bca604c0a8dfb34e65b22214d is 50, key is test_row_0/C:col10/1731147438184/Put/seqid=0 2024-11-09T10:17:19,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742205_1381 (size=12301) 2024-11-09T10:17:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1731147499779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39372 deadline: 1731147499792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:19,808 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,809 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 6fd4a3e490137b74edc89a46b520f093 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:19,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T10:17:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:19,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T10:17:19,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T10:17:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:19,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:20,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147500129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:20,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147500129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:20,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T10:17:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:20,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T10:17:20,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/fcff002bca604c0a8dfb34e65b22214d 2024-11-09T10:17:20,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/c5cba13e9d274fad8f72ee9575a7d14e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c5cba13e9d274fad8f72ee9575a7d14e 2024-11-09T10:17:20,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c5cba13e9d274fad8f72ee9575a7d14e, entries=200, sequenceid=276, filesize=39.0 K 2024-11-09T10:17:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b96f8ea63201456685e5db30c55b25b6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b96f8ea63201456685e5db30c55b25b6 2024-11-09T10:17:20,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b96f8ea63201456685e5db30c55b25b6, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T10:17:20,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/fcff002bca604c0a8dfb34e65b22214d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/fcff002bca604c0a8dfb34e65b22214d 2024-11-09T10:17:20,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/fcff002bca604c0a8dfb34e65b22214d, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T10:17:20,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6fd4a3e490137b74edc89a46b520f093 in 1405ms, sequenceid=276, compaction requested=false 2024-11-09T10:17:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:20,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T10:17:20,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:20,312 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T10:17:20,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110963c7d74360ee460e8064317b57e59b69_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147438911/Put/seqid=0 2024-11-09T10:17:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742206_1382 (size=12454) 2024-11-09T10:17:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:20,358 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110963c7d74360ee460e8064317b57e59b69_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963c7d74360ee460e8064317b57e59b69_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:20,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/6302adb3099d4301a79f012ed0616214, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:20,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/6302adb3099d4301a79f012ed0616214 is 175, key is test_row_0/A:col10/1731147438911/Put/seqid=0 2024-11-09T10:17:20,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742207_1383 (size=31255) 2024-11-09T10:17:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T10:17:20,774 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/6302adb3099d4301a79f012ed0616214 2024-11-09T10:17:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/14a3a0a34e5841f7a93f8afedf62726b is 50, key is test_row_0/B:col10/1731147438911/Put/seqid=0 2024-11-09T10:17:20,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742208_1384 (size=12301) 2024-11-09T10:17:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:21,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:21,221 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/14a3a0a34e5841f7a93f8afedf62726b 2024-11-09T10:17:21,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/6558cde8eb974d9bb9fd5abed0ed831d is 50, key is test_row_0/C:col10/1731147438911/Put/seqid=0 2024-11-09T10:17:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147501223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:21,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742209_1385 (size=12301) 2024-11-09T10:17:21,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147501237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147501349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:21,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147501359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147501568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:21,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147501579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:21,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/6558cde8eb974d9bb9fd5abed0ed831d 2024-11-09T10:17:21,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/6302adb3099d4301a79f012ed0616214 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/6302adb3099d4301a79f012ed0616214 2024-11-09T10:17:21,655 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/6302adb3099d4301a79f012ed0616214, entries=150, sequenceid=293, filesize=30.5 K 2024-11-09T10:17:21,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/14a3a0a34e5841f7a93f8afedf62726b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/14a3a0a34e5841f7a93f8afedf62726b 2024-11-09T10:17:21,662 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/14a3a0a34e5841f7a93f8afedf62726b, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T10:17:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/6558cde8eb974d9bb9fd5abed0ed831d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/6558cde8eb974d9bb9fd5abed0ed831d 2024-11-09T10:17:21,666 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/6558cde8eb974d9bb9fd5abed0ed831d, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T10:17:21,667 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6fd4a3e490137b74edc89a46b520f093 in 1355ms, sequenceid=293, compaction requested=true 2024-11-09T10:17:21,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:21,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:21,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-09T10:17:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-09T10:17:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T10:17:21,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-09T10:17:21,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1460 sec 2024-11-09T10:17:21,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1500 sec 2024-11-09T10:17:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:21,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:17:21,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:21,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:21,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:21,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:21,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:21,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:21,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090c9009cabc8a4bc994fcf6ece669f2e0_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147441184/Put/seqid=0 2024-11-09T10:17:21,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742210_1386 (size=14994) 2024-11-09T10:17:21,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:21,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090c9009cabc8a4bc994fcf6ece669f2e0_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090c9009cabc8a4bc994fcf6ece669f2e0_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:21,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/207aeb1ac0704bbeaf431a7b653cb0ea, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:21,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/207aeb1ac0704bbeaf431a7b653cb0ea is 175, key is test_row_0/A:col10/1731147441184/Put/seqid=0 2024-11-09T10:17:21,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742211_1387 (size=39949) 2024-11-09T10:17:21,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147501929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:21,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147501930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147502038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147502049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147502248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147502269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,335 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/207aeb1ac0704bbeaf431a7b653cb0ea 2024-11-09T10:17:22,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/8e1c096460ff4ffd8b61ffa47bb69226 is 50, key is test_row_0/B:col10/1731147441184/Put/seqid=0 2024-11-09T10:17:22,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742212_1388 (size=12301) 2024-11-09T10:17:22,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/8e1c096460ff4ffd8b61ffa47bb69226 2024-11-09T10:17:22,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e350df3ad5934b10b20550fe1389dc87 is 50, key is test_row_0/C:col10/1731147441184/Put/seqid=0 2024-11-09T10:17:22,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742213_1389 (size=12301) 2024-11-09T10:17:22,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e350df3ad5934b10b20550fe1389dc87 2024-11-09T10:17:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/207aeb1ac0704bbeaf431a7b653cb0ea as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/207aeb1ac0704bbeaf431a7b653cb0ea 2024-11-09T10:17:22,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/207aeb1ac0704bbeaf431a7b653cb0ea, entries=200, sequenceid=312, filesize=39.0 K 2024-11-09T10:17:22,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/8e1c096460ff4ffd8b61ffa47bb69226 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/8e1c096460ff4ffd8b61ffa47bb69226 2024-11-09T10:17:22,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/8e1c096460ff4ffd8b61ffa47bb69226, entries=150, sequenceid=312, filesize=12.0 K 2024-11-09T10:17:22,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e350df3ad5934b10b20550fe1389dc87 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e350df3ad5934b10b20550fe1389dc87 2024-11-09T10:17:22,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e350df3ad5934b10b20550fe1389dc87, entries=150, sequenceid=312, filesize=12.0 K 2024-11-09T10:17:22,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6fd4a3e490137b74edc89a46b520f093 in 523ms, sequenceid=312, compaction requested=true 2024-11-09T10:17:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:22,409 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:22,410 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:22,410 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142854 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:22,410 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/A is initiating minor compaction (all files) 2024-11-09T10:17:22,411 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/A in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:22,411 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e120bdf91d3740c4971cc2f8c63c6f3b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c5cba13e9d274fad8f72ee9575a7d14e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/6302adb3099d4301a79f012ed0616214, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/207aeb1ac0704bbeaf431a7b653cb0ea] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=139.5 K 2024-11-09T10:17:22,411 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:22,411 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e120bdf91d3740c4971cc2f8c63c6f3b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c5cba13e9d274fad8f72ee9575a7d14e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/6302adb3099d4301a79f012ed0616214, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/207aeb1ac0704bbeaf431a7b653cb0ea] 2024-11-09T10:17:22,411 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49650 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:22,411 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/B is initiating minor compaction (all files) 2024-11-09T10:17:22,411 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/B in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:22,411 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/64419861c7eb41eb8e90e3ba05264308, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b96f8ea63201456685e5db30c55b25b6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/14a3a0a34e5841f7a93f8afedf62726b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/8e1c096460ff4ffd8b61ffa47bb69226] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=48.5 K 2024-11-09T10:17:22,412 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e120bdf91d3740c4971cc2f8c63c6f3b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147437440 2024-11-09T10:17:22,412 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 64419861c7eb41eb8e90e3ba05264308, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147437440 2024-11-09T10:17:22,412 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5cba13e9d274fad8f72ee9575a7d14e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147438184 2024-11-09T10:17:22,412 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b96f8ea63201456685e5db30c55b25b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147438184 2024-11-09T10:17:22,412 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6302adb3099d4301a79f012ed0616214, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731147438911 2024-11-09T10:17:22,412 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a3a0a34e5841f7a93f8afedf62726b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731147438911 2024-11-09T10:17:22,412 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 207aeb1ac0704bbeaf431a7b653cb0ea, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731147441184 2024-11-09T10:17:22,413 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e1c096460ff4ffd8b61ffa47bb69226, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731147441184 2024-11-09T10:17:22,431 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:22,433 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#B#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:22,434 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bca59ed79dfa451aba961c940265c579 is 50, key is test_row_0/B:col10/1731147441184/Put/seqid=0 2024-11-09T10:17:22,437 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411095daf8d51f95e41ffb6622edd3d82a618_6fd4a3e490137b74edc89a46b520f093 store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:22,443 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411095daf8d51f95e41ffb6622edd3d82a618_6fd4a3e490137b74edc89a46b520f093, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:22,443 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095daf8d51f95e41ffb6622edd3d82a618_6fd4a3e490137b74edc89a46b520f093 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:22,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742214_1390 (size=12983) 2024-11-09T10:17:22,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742215_1391 (size=4469) 2024-11-09T10:17:22,460 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#A#compaction#324 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:22,460 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/56f50b078ee14db0b0d7630a9424b188 is 175, key is test_row_0/A:col10/1731147441184/Put/seqid=0 2024-11-09T10:17:22,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742216_1392 (size=31937) 2024-11-09T10:17:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:22,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T10:17:22,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:22,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:22,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:22,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:22,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:22,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:22,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098a8cfce775794f9289dfd444f1b8e77d_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147442575/Put/seqid=0 2024-11-09T10:17:22,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742217_1393 (size=14994) 2024-11-09T10:17:22,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147502667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147502684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147502789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147502799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:22,857 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/bca59ed79dfa451aba961c940265c579 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bca59ed79dfa451aba961c940265c579 2024-11-09T10:17:22,862 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/B of 6fd4a3e490137b74edc89a46b520f093 into bca59ed79dfa451aba961c940265c579(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:22,862 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:22,862 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/B, priority=12, startTime=1731147442410; duration=0sec 2024-11-09T10:17:22,862 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:22,862 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:B 2024-11-09T10:17:22,862 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:22,863 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49650 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:22,863 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/C is initiating minor compaction (all files) 2024-11-09T10:17:22,863 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/C in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:22,863 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/f316a05179b149c99f03a29566af655d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/fcff002bca604c0a8dfb34e65b22214d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/6558cde8eb974d9bb9fd5abed0ed831d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e350df3ad5934b10b20550fe1389dc87] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=48.5 K 2024-11-09T10:17:22,864 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting f316a05179b149c99f03a29566af655d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731147437440 2024-11-09T10:17:22,864 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting fcff002bca604c0a8dfb34e65b22214d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731147438184 2024-11-09T10:17:22,864 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 6558cde8eb974d9bb9fd5abed0ed831d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731147438911 2024-11-09T10:17:22,864 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e350df3ad5934b10b20550fe1389dc87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731147441184 2024-11-09T10:17:22,876 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/56f50b078ee14db0b0d7630a9424b188 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/56f50b078ee14db0b0d7630a9424b188 2024-11-09T10:17:22,879 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#C#compaction#326 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:22,879 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/A of 6fd4a3e490137b74edc89a46b520f093 into 56f50b078ee14db0b0d7630a9424b188(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:22,879 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:22,879 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/A, priority=12, startTime=1731147442409; duration=0sec 2024-11-09T10:17:22,879 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:22,879 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:A 2024-11-09T10:17:22,879 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/c2ef730f42414e68852d9c40cf09ea0e is 50, key is test_row_0/C:col10/1731147441184/Put/seqid=0 2024-11-09T10:17:22,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742218_1394 (size=12983) 2024-11-09T10:17:22,887 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/c2ef730f42414e68852d9c40cf09ea0e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/c2ef730f42414e68852d9c40cf09ea0e 2024-11-09T10:17:22,891 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/C of 6fd4a3e490137b74edc89a46b520f093 into c2ef730f42414e68852d9c40cf09ea0e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:22,891 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:22,891 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/C, priority=12, startTime=1731147442410; duration=0sec 2024-11-09T10:17:22,891 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:22,892 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:C 2024-11-09T10:17:22,996 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:23,000 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098a8cfce775794f9289dfd444f1b8e77d_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098a8cfce775794f9289dfd444f1b8e77d_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:23,001 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/80b1a4e3b0ff42a6838dcfa070f12ca8, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:23,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/80b1a4e3b0ff42a6838dcfa070f12ca8 is 175, key is test_row_0/A:col10/1731147442575/Put/seqid=0 2024-11-09T10:17:23,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742219_1395 (size=39949) 2024-11-09T10:17:23,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/80b1a4e3b0ff42a6838dcfa070f12ca8 2024-11-09T10:17:23,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b12aa007fc054fd5a4058d47aef715f5 is 50, key is test_row_0/B:col10/1731147442575/Put/seqid=0 2024-11-09T10:17:23,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147503010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:23,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147503011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:23,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742220_1396 (size=12301) 2024-11-09T10:17:23,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b12aa007fc054fd5a4058d47aef715f5 2024-11-09T10:17:23,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/98033673215a4546b0209883ca84e1bc is 50, key is test_row_0/C:col10/1731147442575/Put/seqid=0 2024-11-09T10:17:23,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742221_1397 (size=12301) 2024-11-09T10:17:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147503329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147503329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:23,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/98033673215a4546b0209883ca84e1bc 2024-11-09T10:17:23,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/80b1a4e3b0ff42a6838dcfa070f12ca8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/80b1a4e3b0ff42a6838dcfa070f12ca8 2024-11-09T10:17:23,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/80b1a4e3b0ff42a6838dcfa070f12ca8, entries=200, sequenceid=326, filesize=39.0 K 2024-11-09T10:17:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/b12aa007fc054fd5a4058d47aef715f5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b12aa007fc054fd5a4058d47aef715f5 2024-11-09T10:17:23,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b12aa007fc054fd5a4058d47aef715f5, entries=150, sequenceid=326, filesize=12.0 K 2024-11-09T10:17:23,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/98033673215a4546b0209883ca84e1bc as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/98033673215a4546b0209883ca84e1bc 2024-11-09T10:17:23,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/98033673215a4546b0209883ca84e1bc, entries=150, sequenceid=326, filesize=12.0 K 2024-11-09T10:17:23,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6fd4a3e490137b74edc89a46b520f093 in 877ms, sequenceid=326, compaction requested=false 2024-11-09T10:17:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T10:17:23,679 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:23,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-09T10:17:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T10:17:23,681 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:23,682 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:23,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T10:17:23,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-09T10:17:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:23,834 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T10:17:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a610885534554deb94a4955d6b5f72c0_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147442651/Put/seqid=0 2024-11-09T10:17:23,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742222_1398 (size=12454) 2024-11-09T10:17:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:23,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:23,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147503889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147503892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T10:17:24,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147504008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:24,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147504009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:24,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147504220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:24,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147504220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:24,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:24,246 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a610885534554deb94a4955d6b5f72c0_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a610885534554deb94a4955d6b5f72c0_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/812ffde205d54ddeb926781b7d0c8e90, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/812ffde205d54ddeb926781b7d0c8e90 is 175, key is test_row_0/A:col10/1731147442651/Put/seqid=0 2024-11-09T10:17:24,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742223_1399 (size=31255) 2024-11-09T10:17:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T10:17:24,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147504539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:24,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147504539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:24,655 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/812ffde205d54ddeb926781b7d0c8e90 2024-11-09T10:17:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/f649fc8142e3488b8abea61eae0f122e is 50, key is test_row_0/B:col10/1731147442651/Put/seqid=0 2024-11-09T10:17:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742224_1400 (size=12301) 2024-11-09T10:17:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T10:17:25,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147505059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147505059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:25,070 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/f649fc8142e3488b8abea61eae0f122e 2024-11-09T10:17:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/b1b48b736a1f41b890e16397681af0eb is 50, key is test_row_0/C:col10/1731147442651/Put/seqid=0 2024-11-09T10:17:25,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742225_1401 (size=12301) 2024-11-09T10:17:25,508 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/b1b48b736a1f41b890e16397681af0eb 2024-11-09T10:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/812ffde205d54ddeb926781b7d0c8e90 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/812ffde205d54ddeb926781b7d0c8e90 2024-11-09T10:17:25,516 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/812ffde205d54ddeb926781b7d0c8e90, entries=150, sequenceid=347, filesize=30.5 K 2024-11-09T10:17:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/f649fc8142e3488b8abea61eae0f122e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f649fc8142e3488b8abea61eae0f122e 2024-11-09T10:17:25,520 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f649fc8142e3488b8abea61eae0f122e, entries=150, sequenceid=347, filesize=12.0 K 2024-11-09T10:17:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/b1b48b736a1f41b890e16397681af0eb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1b48b736a1f41b890e16397681af0eb 2024-11-09T10:17:25,524 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1b48b736a1f41b890e16397681af0eb, entries=150, sequenceid=347, filesize=12.0 K 2024-11-09T10:17:25,525 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6fd4a3e490137b74edc89a46b520f093 in 1691ms, sequenceid=347, compaction requested=true 2024-11-09T10:17:25,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:25,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:25,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-09T10:17:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-09T10:17:25,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-09T10:17:25,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8450 sec 2024-11-09T10:17:25,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8490 sec 2024-11-09T10:17:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T10:17:25,829 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:25,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-09T10:17:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T10:17:25,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:25,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:25,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T10:17:25,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T10:17:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:25,984 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T10:17:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f59ee96e001f48fe8eb4c72a07c9bd55_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147443891/Put/seqid=0 2024-11-09T10:17:25,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742226_1402 (size=12454) 2024-11-09T10:17:26,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. as already flushing 2024-11-09T10:17:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:26,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147506138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T10:17:26,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147506139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147506248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147506261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:26,396 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f59ee96e001f48fe8eb4c72a07c9bd55_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f59ee96e001f48fe8eb4c72a07c9bd55_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a92e77e965d64b378e1e7adcb725e52d, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a92e77e965d64b378e1e7adcb725e52d is 175, key is test_row_0/A:col10/1731147443891/Put/seqid=0 2024-11-09T10:17:26,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742227_1403 (size=31255) 2024-11-09T10:17:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T10:17:26,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147506469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147506488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,750 INFO [Thread-1437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1437. 2024-11-09T10:17:26,750 DEBUG [Thread-1437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:26,750 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,750 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:26,750 INFO [Thread-1439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1439. 2024-11-09T10:17:26,750 DEBUG [Thread-1439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:26,750 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,751 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:26,755 INFO [Thread-1443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1443. 2024-11-09T10:17:26,755 DEBUG [Thread-1443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:26,755 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,755 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:26,762 INFO [Thread-1441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1441. 2024-11-09T10:17:26,762 DEBUG [Thread-1441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:26,762 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,762 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:26,762 INFO [Thread-1445 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1445. 2024-11-09T10:17:26,762 DEBUG [Thread-1445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:26,762 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,762 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:26,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:26,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147506779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,800 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a92e77e965d64b378e1e7adcb725e52d 2024-11-09T10:17:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/432582b1041242d59c271e9e07d0bdd1 is 50, key is test_row_0/B:col10/1731147443891/Put/seqid=0 2024-11-09T10:17:26,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147506809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:26,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742228_1404 (size=12301) 2024-11-09T10:17:26,956 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T10:17:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T10:17:27,212 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/432582b1041242d59c271e9e07d0bdd1 2024-11-09T10:17:27,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e821ad3e898e44c7852bcf6a4434c6cd is 50, key is test_row_0/C:col10/1731147443891/Put/seqid=0 2024-11-09T10:17:27,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742229_1405 (size=12301) 2024-11-09T10:17:27,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39402 deadline: 1731147507289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39406 deadline: 1731147507319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:17:27,622 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e821ad3e898e44c7852bcf6a4434c6cd 2024-11-09T10:17:27,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/a92e77e965d64b378e1e7adcb725e52d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a92e77e965d64b378e1e7adcb725e52d 2024-11-09T10:17:27,629 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a92e77e965d64b378e1e7adcb725e52d, entries=150, sequenceid=361, filesize=30.5 K 2024-11-09T10:17:27,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/432582b1041242d59c271e9e07d0bdd1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/432582b1041242d59c271e9e07d0bdd1 2024-11-09T10:17:27,633 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/432582b1041242d59c271e9e07d0bdd1, entries=150, sequenceid=361, filesize=12.0 K 2024-11-09T10:17:27,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/e821ad3e898e44c7852bcf6a4434c6cd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e821ad3e898e44c7852bcf6a4434c6cd 2024-11-09T10:17:27,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e821ad3e898e44c7852bcf6a4434c6cd, entries=150, sequenceid=361, filesize=12.0 K 2024-11-09T10:17:27,638 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6fd4a3e490137b74edc89a46b520f093 in 1653ms, sequenceid=361, compaction requested=true 2024-11-09T10:17:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-09T10:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-09T10:17:27,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-09T10:17:27,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-11-09T10:17:27,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8120 sec 2024-11-09T10:17:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T10:17:27,980 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:28,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:17:28,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:28,301 INFO [Thread-1426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1426. 2024-11-09T10:17:28,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:28,301 DEBUG [Thread-1426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:28,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:28,301 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:28,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:28,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:28,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:28,301 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:28,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:28,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096de0bd4ea1174fea9c6f3afb6229d4b3_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147446138/Put/seqid=0 2024-11-09T10:17:28,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742230_1406 (size=12454) 2024-11-09T10:17:28,340 INFO [Thread-1434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1434. 2024-11-09T10:17:28,340 DEBUG [Thread-1434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:28,340 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:28,340 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:28,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:28,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:28,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096de0bd4ea1174fea9c6f3afb6229d4b3_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096de0bd4ea1174fea9c6f3afb6229d4b3_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:28,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/bbf7881c1bdb430385c48b5a57d32d6d, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:28,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/bbf7881c1bdb430385c48b5a57d32d6d is 175, key is test_row_0/A:col10/1731147446138/Put/seqid=0 2024-11-09T10:17:28,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742231_1407 (size=31255) 2024-11-09T10:17:29,117 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/bbf7881c1bdb430385c48b5a57d32d6d 2024-11-09T10:17:29,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/52d6c823d8ce4485a53c2ed7b23eec94 is 50, key is test_row_0/B:col10/1731147446138/Put/seqid=0 2024-11-09T10:17:29,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742232_1408 (size=12301) 2024-11-09T10:17:29,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/52d6c823d8ce4485a53c2ed7b23eec94 2024-11-09T10:17:29,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/24a42c998b63470dafacb1d8b25451ca is 50, key is test_row_0/C:col10/1731147446138/Put/seqid=0 2024-11-09T10:17:29,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742233_1409 (size=12301) 2024-11-09T10:17:29,830 INFO [Thread-1430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1430. 2024-11-09T10:17:29,830 DEBUG [Thread-1430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:29,830 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,830 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:29,841 INFO [Thread-1432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1432. 2024-11-09T10:17:29,841 DEBUG [Thread-1432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:29,841 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,841 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:29,870 INFO [Thread-1428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1428. 2024-11-09T10:17:29,870 DEBUG [Thread-1428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:29,870 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,871 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 982 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2946 rows 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 986 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2958 rows 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 987 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2961 rows 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 984 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2952 rows 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 981 2024-11-09T10:17:29,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2943 rows 2024-11-09T10:17:29,871 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:17:29,871 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:17:29,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:29,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:29,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T10:17:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T10:17:29,876 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147449875"}]},"ts":"1731147449875"} 2024-11-09T10:17:29,877 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T10:17:29,877 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T10:17:29,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:17:29,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, UNASSIGN}] 2024-11-09T10:17:29,880 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, UNASSIGN 2024-11-09T10:17:29,881 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:29,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, UNASSIGN because future has completed 2024-11-09T10:17:29,883 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:17:29,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:17:29,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/24a42c998b63470dafacb1d8b25451ca 2024-11-09T10:17:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/bbf7881c1bdb430385c48b5a57d32d6d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/bbf7881c1bdb430385c48b5a57d32d6d 2024-11-09T10:17:29,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/bbf7881c1bdb430385c48b5a57d32d6d, entries=150, sequenceid=380, filesize=30.5 K 2024-11-09T10:17:29,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/52d6c823d8ce4485a53c2ed7b23eec94 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/52d6c823d8ce4485a53c2ed7b23eec94 2024-11-09T10:17:29,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/52d6c823d8ce4485a53c2ed7b23eec94, entries=150, sequenceid=380, filesize=12.0 K 2024-11-09T10:17:29,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/24a42c998b63470dafacb1d8b25451ca as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/24a42c998b63470dafacb1d8b25451ca 2024-11-09T10:17:29,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/24a42c998b63470dafacb1d8b25451ca, entries=150, sequenceid=380, filesize=12.0 K 2024-11-09T10:17:29,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for 6fd4a3e490137b74edc89a46b520f093 in 1652ms, sequenceid=380, compaction requested=true 2024-11-09T10:17:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fd4a3e490137b74edc89a46b520f093:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:29,952 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T10:17:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:29,952 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165651 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62187 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/A is initiating minor compaction (all files) 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 6fd4a3e490137b74edc89a46b520f093/B is initiating minor compaction (all files) 2024-11-09T10:17:29,954 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/A in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:29,954 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fd4a3e490137b74edc89a46b520f093/B in TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:29,954 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/56f50b078ee14db0b0d7630a9424b188, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/80b1a4e3b0ff42a6838dcfa070f12ca8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/812ffde205d54ddeb926781b7d0c8e90, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a92e77e965d64b378e1e7adcb725e52d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/bbf7881c1bdb430385c48b5a57d32d6d] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=161.8 K 2024-11-09T10:17:29,954 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bca59ed79dfa451aba961c940265c579, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b12aa007fc054fd5a4058d47aef715f5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f649fc8142e3488b8abea61eae0f122e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/432582b1041242d59c271e9e07d0bdd1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/52d6c823d8ce4485a53c2ed7b23eec94] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp, totalSize=60.7 K 2024-11-09T10:17:29,954 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/56f50b078ee14db0b0d7630a9424b188, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/80b1a4e3b0ff42a6838dcfa070f12ca8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/812ffde205d54ddeb926781b7d0c8e90, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a92e77e965d64b378e1e7adcb725e52d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/bbf7881c1bdb430385c48b5a57d32d6d] 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bca59ed79dfa451aba961c940265c579, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731147441184 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f50b078ee14db0b0d7630a9424b188, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731147441184 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b12aa007fc054fd5a4058d47aef715f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731147441928 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b1a4e3b0ff42a6838dcfa070f12ca8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731147441922 2024-11-09T10:17:29,954 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting f649fc8142e3488b8abea61eae0f122e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147442644 2024-11-09T10:17:29,955 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 812ffde205d54ddeb926781b7d0c8e90, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731147442644 2024-11-09T10:17:29,955 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 432582b1041242d59c271e9e07d0bdd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731147443884 2024-11-09T10:17:29,955 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a92e77e965d64b378e1e7adcb725e52d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731147443884 2024-11-09T10:17:29,955 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d6c823d8ce4485a53c2ed7b23eec94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731147446126 2024-11-09T10:17:29,955 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf7881c1bdb430385c48b5a57d32d6d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731147446126 2024-11-09T10:17:29,968 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#B#compaction#338 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:29,968 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/7e781dd6c6cf487988358b22a541fa02 is 50, key is test_row_0/B:col10/1731147446138/Put/seqid=0 2024-11-09T10:17:29,969 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:29,971 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411095a6cdd1e22e449e6ac64a0aae01fc9d1_6fd4a3e490137b74edc89a46b520f093 store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742234_1410 (size=13153) 2024-11-09T10:17:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T10:17:29,995 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411095a6cdd1e22e449e6ac64a0aae01fc9d1_6fd4a3e490137b74edc89a46b520f093, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:29,995 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095a6cdd1e22e449e6ac64a0aae01fc9d1_6fd4a3e490137b74edc89a46b520f093 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:29,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742235_1411 (size=4469) 2024-11-09T10:17:30,036 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:30,036 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:17:30,036 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 6fd4a3e490137b74edc89a46b520f093, disabling compactions & flushes 2024-11-09T10:17:30,036 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T10:17:30,377 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/7e781dd6c6cf487988358b22a541fa02 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7e781dd6c6cf487988358b22a541fa02 2024-11-09T10:17:30,381 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/B of 6fd4a3e490137b74edc89a46b520f093 into 7e781dd6c6cf487988358b22a541fa02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:30,381 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:30,381 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/B, priority=11, startTime=1731147449952; duration=0sec 2024-11-09T10:17:30,381 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:30,381 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:B 2024-11-09T10:17:30,381 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. because compaction request was cancelled 2024-11-09T10:17:30,381 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:C 2024-11-09T10:17:30,399 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fd4a3e490137b74edc89a46b520f093#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T10:17:30,400 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/5ce90ca8fdd14dae8ee718f70cdf44f4 is 175, key is test_row_0/A:col10/1731147446138/Put/seqid=0 2024-11-09T10:17:30,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742236_1412 (size=32107) 2024-11-09T10:17:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T10:17:30,810 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/5ce90ca8fdd14dae8ee718f70cdf44f4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/5ce90ca8fdd14dae8ee718f70cdf44f4 2024-11-09T10:17:30,815 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6fd4a3e490137b74edc89a46b520f093/A of 6fd4a3e490137b74edc89a46b520f093 into 5ce90ca8fdd14dae8ee718f70cdf44f4(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:30,815 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fd4a3e490137b74edc89a46b520f093: 2024-11-09T10:17:30,815 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:30,815 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., storeName=6fd4a3e490137b74edc89a46b520f093/A, priority=11, startTime=1731147449952; duration=0sec 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. after waiting 0 ms 2024-11-09T10:17:30,815 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:30,815 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fd4a3e490137b74edc89a46b520f093:A 2024-11-09T10:17:30,815 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 6fd4a3e490137b74edc89a46b520f093 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=A 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=B 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fd4a3e490137b74edc89a46b520f093, store=C 2024-11-09T10:17:30,815 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:30,820 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109266dd979e80b49989b7e1d418f4d71d9_6fd4a3e490137b74edc89a46b520f093 is 50, key is test_row_0/A:col10/1731147449869/Put/seqid=0 2024-11-09T10:17:30,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742237_1413 (size=9914) 2024-11-09T10:17:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T10:17:31,148 DEBUG [regionserver/e067b2c2c7b3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-11-09T10:17:31,150 WARN [regionserver/e067b2c2c7b3:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-09T10:17:31,151 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a8a2ab7f349d4dd79f414cec136dfaa8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/2f69e679f045460ca985c8ad41ee131d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e5d29d734f234e5984c53a5d6e1089a5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/fad1a259bf8b4c37b5a5f09a40609b5a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/40666d3300074249a5935e994a8659ce, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/b218dc27ed774542bf39e5b4ee3ec586, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c1f0816f070a41da9620491afd622ddb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/da7d14e49a1e4909928a04cc0015239c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/197fcd935ceb45c5b782c03949846484, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/be39acacbb4b4d3e96a98fc093d6786f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/113e079313c14ca9ba9d73b4dd88b33e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/036aea2d1f904c50854f1a4469f4edd3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/317b0796ab174cadaf976ba4910e82a8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/cf664d1607454338a015c606a662df4b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/25a39859496f4601af246641d64bf8e1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/027a3f2b5f02427f96a9d6587b55b734, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e120bdf91d3740c4971cc2f8c63c6f3b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c5cba13e9d274fad8f72ee9575a7d14e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/6302adb3099d4301a79f012ed0616214, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/207aeb1ac0704bbeaf431a7b653cb0ea, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/56f50b078ee14db0b0d7630a9424b188, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/80b1a4e3b0ff42a6838dcfa070f12ca8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/812ffde205d54ddeb926781b7d0c8e90, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a92e77e965d64b378e1e7adcb725e52d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/bbf7881c1bdb430385c48b5a57d32d6d] to archive 2024-11-09T10:17:31,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:31,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a8a2ab7f349d4dd79f414cec136dfaa8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a8a2ab7f349d4dd79f414cec136dfaa8 2024-11-09T10:17:31,154 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/2f69e679f045460ca985c8ad41ee131d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/2f69e679f045460ca985c8ad41ee131d 2024-11-09T10:17:31,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e5d29d734f234e5984c53a5d6e1089a5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e5d29d734f234e5984c53a5d6e1089a5 2024-11-09T10:17:31,156 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/fad1a259bf8b4c37b5a5f09a40609b5a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/fad1a259bf8b4c37b5a5f09a40609b5a 2024-11-09T10:17:31,156 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/40666d3300074249a5935e994a8659ce to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/40666d3300074249a5935e994a8659ce 2024-11-09T10:17:31,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/b218dc27ed774542bf39e5b4ee3ec586 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/b218dc27ed774542bf39e5b4ee3ec586 2024-11-09T10:17:31,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c1f0816f070a41da9620491afd622ddb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c1f0816f070a41da9620491afd622ddb 2024-11-09T10:17:31,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/da7d14e49a1e4909928a04cc0015239c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/da7d14e49a1e4909928a04cc0015239c 2024-11-09T10:17:31,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/197fcd935ceb45c5b782c03949846484 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/197fcd935ceb45c5b782c03949846484 2024-11-09T10:17:31,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/be39acacbb4b4d3e96a98fc093d6786f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/be39acacbb4b4d3e96a98fc093d6786f 2024-11-09T10:17:31,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/113e079313c14ca9ba9d73b4dd88b33e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/113e079313c14ca9ba9d73b4dd88b33e 2024-11-09T10:17:31,162 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/036aea2d1f904c50854f1a4469f4edd3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/036aea2d1f904c50854f1a4469f4edd3 2024-11-09T10:17:31,163 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/317b0796ab174cadaf976ba4910e82a8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/317b0796ab174cadaf976ba4910e82a8 2024-11-09T10:17:31,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/cf664d1607454338a015c606a662df4b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/cf664d1607454338a015c606a662df4b 2024-11-09T10:17:31,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/25a39859496f4601af246641d64bf8e1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/25a39859496f4601af246641d64bf8e1 2024-11-09T10:17:31,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/027a3f2b5f02427f96a9d6587b55b734 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/027a3f2b5f02427f96a9d6587b55b734 2024-11-09T10:17:31,166 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e120bdf91d3740c4971cc2f8c63c6f3b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/e120bdf91d3740c4971cc2f8c63c6f3b 2024-11-09T10:17:31,167 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c5cba13e9d274fad8f72ee9575a7d14e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/c5cba13e9d274fad8f72ee9575a7d14e 2024-11-09T10:17:31,168 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/6302adb3099d4301a79f012ed0616214 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/6302adb3099d4301a79f012ed0616214 2024-11-09T10:17:31,168 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/207aeb1ac0704bbeaf431a7b653cb0ea to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/207aeb1ac0704bbeaf431a7b653cb0ea 2024-11-09T10:17:31,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/56f50b078ee14db0b0d7630a9424b188 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/56f50b078ee14db0b0d7630a9424b188 2024-11-09T10:17:31,170 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/80b1a4e3b0ff42a6838dcfa070f12ca8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/80b1a4e3b0ff42a6838dcfa070f12ca8 2024-11-09T10:17:31,171 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/812ffde205d54ddeb926781b7d0c8e90 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/812ffde205d54ddeb926781b7d0c8e90 2024-11-09T10:17:31,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a92e77e965d64b378e1e7adcb725e52d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/a92e77e965d64b378e1e7adcb725e52d 2024-11-09T10:17:31,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/bbf7881c1bdb430385c48b5a57d32d6d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/bbf7881c1bdb430385c48b5a57d32d6d 2024-11-09T10:17:31,177 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bf564a5debbb46698b732e3a86871378, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/666e0eb10c6746dca198a8e6a4e9274f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f187ad80acef4a40ade29d75e084e5ce, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/5d77a892c02f4dab9465f4eb09677c05, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b843589417004a92a6e301cdbe926051, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/d6f6cbfc4b5044d68714a7da29f3979a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/a66ffed8fdf144879580c6cc64182b16, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aa17fcc0ec384f028df4e8196df84077, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aaa0339c74f04394a30ddca91d1ce511, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/ff17b9bd50c8487c9f4ce5af2f24022a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bee11505efcd49e8a6f9260a2ccd2af4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/c400ebd7e72540eda76e33e5d4a30a6f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7b8c3e58ad3849c18a272abff4a419fa, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bc8f7c2514864628af95d40d3fadd1be, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/638ab9f0661048338da4b36089cbe110, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/64419861c7eb41eb8e90e3ba05264308, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/07290800757a4793a69741b36e48ab0d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b96f8ea63201456685e5db30c55b25b6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/14a3a0a34e5841f7a93f8afedf62726b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bca59ed79dfa451aba961c940265c579, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/8e1c096460ff4ffd8b61ffa47bb69226, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b12aa007fc054fd5a4058d47aef715f5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f649fc8142e3488b8abea61eae0f122e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/432582b1041242d59c271e9e07d0bdd1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/52d6c823d8ce4485a53c2ed7b23eec94] to archive 2024-11-09T10:17:31,178 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:31,179 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bf564a5debbb46698b732e3a86871378 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bf564a5debbb46698b732e3a86871378 2024-11-09T10:17:31,180 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/666e0eb10c6746dca198a8e6a4e9274f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/666e0eb10c6746dca198a8e6a4e9274f 2024-11-09T10:17:31,180 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f187ad80acef4a40ade29d75e084e5ce to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f187ad80acef4a40ade29d75e084e5ce 2024-11-09T10:17:31,181 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/5d77a892c02f4dab9465f4eb09677c05 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/5d77a892c02f4dab9465f4eb09677c05 2024-11-09T10:17:31,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b843589417004a92a6e301cdbe926051 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b843589417004a92a6e301cdbe926051 2024-11-09T10:17:31,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/d6f6cbfc4b5044d68714a7da29f3979a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/d6f6cbfc4b5044d68714a7da29f3979a 2024-11-09T10:17:31,183 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/a66ffed8fdf144879580c6cc64182b16 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/a66ffed8fdf144879580c6cc64182b16 2024-11-09T10:17:31,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aa17fcc0ec384f028df4e8196df84077 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aa17fcc0ec384f028df4e8196df84077 2024-11-09T10:17:31,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aaa0339c74f04394a30ddca91d1ce511 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/aaa0339c74f04394a30ddca91d1ce511 2024-11-09T10:17:31,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/ff17b9bd50c8487c9f4ce5af2f24022a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/ff17b9bd50c8487c9f4ce5af2f24022a 2024-11-09T10:17:31,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bee11505efcd49e8a6f9260a2ccd2af4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bee11505efcd49e8a6f9260a2ccd2af4 2024-11-09T10:17:31,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/c400ebd7e72540eda76e33e5d4a30a6f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/c400ebd7e72540eda76e33e5d4a30a6f 2024-11-09T10:17:31,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7b8c3e58ad3849c18a272abff4a419fa to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7b8c3e58ad3849c18a272abff4a419fa 2024-11-09T10:17:31,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bc8f7c2514864628af95d40d3fadd1be to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bc8f7c2514864628af95d40d3fadd1be 2024-11-09T10:17:31,189 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/638ab9f0661048338da4b36089cbe110 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/638ab9f0661048338da4b36089cbe110 2024-11-09T10:17:31,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/64419861c7eb41eb8e90e3ba05264308 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/64419861c7eb41eb8e90e3ba05264308 2024-11-09T10:17:31,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/07290800757a4793a69741b36e48ab0d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/07290800757a4793a69741b36e48ab0d 2024-11-09T10:17:31,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b96f8ea63201456685e5db30c55b25b6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b96f8ea63201456685e5db30c55b25b6 2024-11-09T10:17:31,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/14a3a0a34e5841f7a93f8afedf62726b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/14a3a0a34e5841f7a93f8afedf62726b 2024-11-09T10:17:31,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bca59ed79dfa451aba961c940265c579 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/bca59ed79dfa451aba961c940265c579 2024-11-09T10:17:31,194 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/8e1c096460ff4ffd8b61ffa47bb69226 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/8e1c096460ff4ffd8b61ffa47bb69226 2024-11-09T10:17:31,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b12aa007fc054fd5a4058d47aef715f5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/b12aa007fc054fd5a4058d47aef715f5 2024-11-09T10:17:31,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f649fc8142e3488b8abea61eae0f122e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/f649fc8142e3488b8abea61eae0f122e 2024-11-09T10:17:31,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/432582b1041242d59c271e9e07d0bdd1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/432582b1041242d59c271e9e07d0bdd1 2024-11-09T10:17:31,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/52d6c823d8ce4485a53c2ed7b23eec94 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/52d6c823d8ce4485a53c2ed7b23eec94 2024-11-09T10:17:31,201 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a4d080c0cc5b4cfca0b7ced2c59457f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/206240e3aff340469f79eb583f0a9e03, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1442b8f52eab450f8db0146f6babdfa6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dcd08dfcceba49b5a823811c964b4103, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/64c31e9098834665bd20e37ad298d6a0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1e5d80170a3f46e7ad26daa91846b9f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/01c807d4d2ca474187f024fd3b7e69d8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/47c293ef7b504dcd98fcb9838bc7eb66, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a93148e42f3343ab8a3ea05a41a122c1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/36b6d43f839843a3b04becdbe4bfcbad, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1f6c73eeb6f45309313c00ba03e5e93, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dfbd5db7cdad42c3bbedfca3951d613e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e1758c2e08db4743a0316776d1669fdc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/22c0ccf1addf4b9cb77f1eac025cada3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/9a9a293ad29546c49e325983dc563cfb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/f316a05179b149c99f03a29566af655d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/0dc53b9d511d47c1a346637b27bf4901, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/fcff002bca604c0a8dfb34e65b22214d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/6558cde8eb974d9bb9fd5abed0ed831d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e350df3ad5934b10b20550fe1389dc87] to archive 2024-11-09T10:17:31,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:31,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a4d080c0cc5b4cfca0b7ced2c59457f1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a4d080c0cc5b4cfca0b7ced2c59457f1 2024-11-09T10:17:31,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/206240e3aff340469f79eb583f0a9e03 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/206240e3aff340469f79eb583f0a9e03 2024-11-09T10:17:31,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1442b8f52eab450f8db0146f6babdfa6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1442b8f52eab450f8db0146f6babdfa6 2024-11-09T10:17:31,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dcd08dfcceba49b5a823811c964b4103 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dcd08dfcceba49b5a823811c964b4103 2024-11-09T10:17:31,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/64c31e9098834665bd20e37ad298d6a0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/64c31e9098834665bd20e37ad298d6a0 2024-11-09T10:17:31,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1e5d80170a3f46e7ad26daa91846b9f1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/1e5d80170a3f46e7ad26daa91846b9f1 2024-11-09T10:17:31,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/01c807d4d2ca474187f024fd3b7e69d8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/01c807d4d2ca474187f024fd3b7e69d8 2024-11-09T10:17:31,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/47c293ef7b504dcd98fcb9838bc7eb66 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/47c293ef7b504dcd98fcb9838bc7eb66 2024-11-09T10:17:31,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a93148e42f3343ab8a3ea05a41a122c1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/a93148e42f3343ab8a3ea05a41a122c1 2024-11-09T10:17:31,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/36b6d43f839843a3b04becdbe4bfcbad to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/36b6d43f839843a3b04becdbe4bfcbad 2024-11-09T10:17:31,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1f6c73eeb6f45309313c00ba03e5e93 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1f6c73eeb6f45309313c00ba03e5e93 2024-11-09T10:17:31,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dfbd5db7cdad42c3bbedfca3951d613e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/dfbd5db7cdad42c3bbedfca3951d613e 2024-11-09T10:17:31,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e1758c2e08db4743a0316776d1669fdc to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e1758c2e08db4743a0316776d1669fdc 2024-11-09T10:17:31,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/22c0ccf1addf4b9cb77f1eac025cada3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/22c0ccf1addf4b9cb77f1eac025cada3 2024-11-09T10:17:31,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/9a9a293ad29546c49e325983dc563cfb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/9a9a293ad29546c49e325983dc563cfb 2024-11-09T10:17:31,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/f316a05179b149c99f03a29566af655d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/f316a05179b149c99f03a29566af655d 2024-11-09T10:17:31,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/0dc53b9d511d47c1a346637b27bf4901 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/0dc53b9d511d47c1a346637b27bf4901 2024-11-09T10:17:31,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/fcff002bca604c0a8dfb34e65b22214d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/fcff002bca604c0a8dfb34e65b22214d 2024-11-09T10:17:31,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/6558cde8eb974d9bb9fd5abed0ed831d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/6558cde8eb974d9bb9fd5abed0ed831d 2024-11-09T10:17:31,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e350df3ad5934b10b20550fe1389dc87 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e350df3ad5934b10b20550fe1389dc87 2024-11-09T10:17:31,223 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:31,226 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109266dd979e80b49989b7e1d418f4d71d9_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109266dd979e80b49989b7e1d418f4d71d9_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:31,227 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/96d18219a35e4bc79ba4ca792cccac7c, store: [table=TestAcidGuarantees family=A region=6fd4a3e490137b74edc89a46b520f093] 2024-11-09T10:17:31,228 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/96d18219a35e4bc79ba4ca792cccac7c is 175, key is test_row_0/A:col10/1731147449869/Put/seqid=0 2024-11-09T10:17:31,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742238_1414 (size=22561) 2024-11-09T10:17:31,633 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/96d18219a35e4bc79ba4ca792cccac7c 2024-11-09T10:17:31,645 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/17b5d8187b52446fb3f781b3e741bcc3 is 50, key is test_row_0/B:col10/1731147449869/Put/seqid=0 2024-11-09T10:17:31,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742239_1415 (size=9857) 2024-11-09T10:17:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T10:17:32,050 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/17b5d8187b52446fb3f781b3e741bcc3 2024-11-09T10:17:32,057 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/66a7e957ba9d4b5d975cec8b7c1489b0 is 50, key is test_row_0/C:col10/1731147449869/Put/seqid=0 2024-11-09T10:17:32,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742240_1416 (size=9857) 2024-11-09T10:17:32,462 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/66a7e957ba9d4b5d975cec8b7c1489b0 2024-11-09T10:17:32,467 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/A/96d18219a35e4bc79ba4ca792cccac7c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/96d18219a35e4bc79ba4ca792cccac7c 2024-11-09T10:17:32,471 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/96d18219a35e4bc79ba4ca792cccac7c, entries=100, sequenceid=389, filesize=22.0 K 2024-11-09T10:17:32,471 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/B/17b5d8187b52446fb3f781b3e741bcc3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/17b5d8187b52446fb3f781b3e741bcc3 2024-11-09T10:17:32,474 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/17b5d8187b52446fb3f781b3e741bcc3, entries=100, sequenceid=389, filesize=9.6 K 2024-11-09T10:17:32,475 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/.tmp/C/66a7e957ba9d4b5d975cec8b7c1489b0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/66a7e957ba9d4b5d975cec8b7c1489b0 2024-11-09T10:17:32,478 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/66a7e957ba9d4b5d975cec8b7c1489b0, entries=100, sequenceid=389, filesize=9.6 K 2024-11-09T10:17:32,479 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6fd4a3e490137b74edc89a46b520f093 in 1664ms, sequenceid=389, compaction requested=true 2024-11-09T10:17:32,487 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/recovered.edits/392.seqid, newMaxSeqId=392, maxSeqId=4 2024-11-09T10:17:32,488 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. 2024-11-09T10:17:32,488 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 6fd4a3e490137b74edc89a46b520f093: Waiting for close lock at 1731147450036Running coprocessor pre-close hooks at 1731147450036Disabling compacts and flushes for region at 1731147450036Disabling writes for close at 1731147450815 (+779 ms)Obtaining lock to block concurrent updates at 1731147450815Preparing flush snapshotting stores in 6fd4a3e490137b74edc89a46b520f093 at 1731147450815Finished memstore snapshotting TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731147450815Flushing stores of TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093. at 1731147450816 (+1 ms)Flushing 6fd4a3e490137b74edc89a46b520f093/A: creating writer at 1731147450816Flushing mob file 6fd4a3e490137b74edc89a46b520f093/A: appending metadata at 1731147450820 (+4 ms)Flushing mob file 6fd4a3e490137b74edc89a46b520f093/A: closing flushed file at 1731147450820Flushing 6fd4a3e490137b74edc89a46b520f093/A: appending metadata at 1731147451227 (+407 ms)Flushing 6fd4a3e490137b74edc89a46b520f093/A: closing flushed file at 1731147451227Flushing 6fd4a3e490137b74edc89a46b520f093/B: creating writer at 1731147451642 (+415 ms)Flushing 6fd4a3e490137b74edc89a46b520f093/B: appending metadata at 1731147451645 (+3 ms)Flushing 6fd4a3e490137b74edc89a46b520f093/B: closing flushed file at 1731147451645Flushing 6fd4a3e490137b74edc89a46b520f093/C: creating writer at 1731147452054 (+409 ms)Flushing 6fd4a3e490137b74edc89a46b520f093/C: appending metadata at 1731147452056 (+2 ms)Flushing 6fd4a3e490137b74edc89a46b520f093/C: closing flushed file at 1731147452056Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16720ff6: reopening flushed file at 1731147452466 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a177334: reopening flushed file at 1731147452471 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@657c1ee7: reopening flushed file at 1731147452474 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6fd4a3e490137b74edc89a46b520f093 in 1664ms, sequenceid=389, compaction requested=true at 1731147452479 (+5 ms)Writing region close event to WAL at 1731147452484 (+5 ms)Running coprocessor post-close hooks at 1731147452488 (+4 ms)Closed at 1731147452488 2024-11-09T10:17:32,490 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:32,490 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6fd4a3e490137b74edc89a46b520f093, regionState=CLOSED 2024-11-09T10:17:32,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:17:32,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-09T10:17:32,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fd4a3e490137b74edc89a46b520f093, server=e067b2c2c7b3,38427,1731147328756 in 2.6110 sec 2024-11-09T10:17:32,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-09T10:17:32,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fd4a3e490137b74edc89a46b520f093, UNASSIGN in 2.6170 sec 2024-11-09T10:17:32,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-09T10:17:32,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6200 sec 2024-11-09T10:17:32,500 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147452500"}]},"ts":"1731147452500"} 2024-11-09T10:17:32,502 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T10:17:32,502 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T10:17:32,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6300 sec 2024-11-09T10:17:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T10:17:34,040 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:34,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T10:17:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:34,045 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T10:17:34,046 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:34,049 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,053 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/recovered.edits] 2024-11-09T10:17:34,056 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/5ce90ca8fdd14dae8ee718f70cdf44f4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/5ce90ca8fdd14dae8ee718f70cdf44f4 2024-11-09T10:17:34,057 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/96d18219a35e4bc79ba4ca792cccac7c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/A/96d18219a35e4bc79ba4ca792cccac7c 2024-11-09T10:17:34,059 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/17b5d8187b52446fb3f781b3e741bcc3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/17b5d8187b52446fb3f781b3e741bcc3 2024-11-09T10:17:34,060 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7e781dd6c6cf487988358b22a541fa02 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/B/7e781dd6c6cf487988358b22a541fa02 2024-11-09T10:17:34,061 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/24a42c998b63470dafacb1d8b25451ca to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/24a42c998b63470dafacb1d8b25451ca 2024-11-09T10:17:34,062 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/66a7e957ba9d4b5d975cec8b7c1489b0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/66a7e957ba9d4b5d975cec8b7c1489b0 2024-11-09T10:17:34,062 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/98033673215a4546b0209883ca84e1bc to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/98033673215a4546b0209883ca84e1bc 2024-11-09T10:17:34,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1b48b736a1f41b890e16397681af0eb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/b1b48b736a1f41b890e16397681af0eb 2024-11-09T10:17:34,064 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/c2ef730f42414e68852d9c40cf09ea0e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/c2ef730f42414e68852d9c40cf09ea0e 2024-11-09T10:17:34,064 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e821ad3e898e44c7852bcf6a4434c6cd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/C/e821ad3e898e44c7852bcf6a4434c6cd 2024-11-09T10:17:34,067 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/recovered.edits/392.seqid to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093/recovered.edits/392.seqid 2024-11-09T10:17:34,067 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,067 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T10:17:34,068 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T10:17:34,068 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T10:17:34,071 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090b393026a0f5428ca322bca51a8288b4_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090b393026a0f5428ca322bca51a8288b4_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,072 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090c9009cabc8a4bc994fcf6ece669f2e0_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090c9009cabc8a4bc994fcf6ece669f2e0_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,073 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090e5550470f484be591e313b99e9e3a47_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090e5550470f484be591e313b99e9e3a47_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,074 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109266dd979e80b49989b7e1d418f4d71d9_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109266dd979e80b49989b7e1d418f4d71d9_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,075 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110935893d711ad943888a37bae768c4e5bc_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110935893d711ad943888a37bae768c4e5bc_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,076 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110947bfd70b10ae4942a06446c886fd3aa5_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110947bfd70b10ae4942a06446c886fd3aa5_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,077 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fc94abad1b145d88945db301f5b98e7_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fc94abad1b145d88945db301f5b98e7_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,078 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110956d69e96dbcb4ac0bef75f0cb55442f3_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110956d69e96dbcb4ac0bef75f0cb55442f3_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,079 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961aa1109a6f24668bf2bee6accc88f28_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961aa1109a6f24668bf2bee6accc88f28_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,080 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963c7d74360ee460e8064317b57e59b69_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963c7d74360ee460e8064317b57e59b69_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,081 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109660341de5ada48c28314f35b762fef20_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109660341de5ada48c28314f35b762fef20_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,082 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096de0bd4ea1174fea9c6f3afb6229d4b3_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096de0bd4ea1174fea9c6f3afb6229d4b3_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,082 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096fc19732cb93443f894f38e07a9092a4_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096fc19732cb93443f894f38e07a9092a4_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,083 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098a8cfce775794f9289dfd444f1b8e77d_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098a8cfce775794f9289dfd444f1b8e77d_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,084 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a610885534554deb94a4955d6b5f72c0_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a610885534554deb94a4955d6b5f72c0_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,085 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b6f12f8b96d1491c8ca2e76c27e41993_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b6f12f8b96d1491c8ca2e76c27e41993_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,086 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c1d664e9862944d8845840c417a1c285_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c1d664e9862944d8845840c417a1c285_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,087 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c352ba96005d4d8a83e49fbcf0a09cae_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c352ba96005d4d8a83e49fbcf0a09cae_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,088 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e2d5b07d0b0b4430b84303a69a3591c3_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e2d5b07d0b0b4430b84303a69a3591c3_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,089 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f59ee96e001f48fe8eb4c72a07c9bd55_6fd4a3e490137b74edc89a46b520f093 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f59ee96e001f48fe8eb4c72a07c9bd55_6fd4a3e490137b74edc89a46b520f093 2024-11-09T10:17:34,089 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T10:17:34,091 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:34,093 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T10:17:34,095 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T10:17:34,096 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:34,097 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T10:17:34,097 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731147454097"}]},"ts":"9223372036854775807"} 2024-11-09T10:17:34,099 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T10:17:34,099 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6fd4a3e490137b74edc89a46b520f093, NAME => 'TestAcidGuarantees,,1731147423989.6fd4a3e490137b74edc89a46b520f093.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T10:17:34,099 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T10:17:34,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731147454099"}]},"ts":"9223372036854775807"} 2024-11-09T10:17:34,102 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T10:17:34,102 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:34,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-09T10:17:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T10:17:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T10:17:34,150 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:34,158 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 246) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1386229718_22 at /127.0.0.1:60180 [Waiting for operation #300] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1452801290_22 at /127.0.0.1:60190 [Waiting for operation #310] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1386229718_22 at /127.0.0.1:60188 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1452801290_22 at /127.0.0.1:59838 [Waiting for operation #432] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/e067b2c2c7b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=553 (was 515) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1848 (was 1171) - AvailableMemoryMB LEAK? - 2024-11-09T10:17:34,165 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=553, ProcessCount=11, AvailableMemoryMB=1848 2024-11-09T10:17:34,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:17:34,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:17:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:34,169 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T10:17:34,169 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:34,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-09T10:17:34,169 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T10:17:34,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-09T10:17:34,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742241_1417 (size=1061) 2024-11-09T10:17:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-09T10:17:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-09T10:17:34,576 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:17:34,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742242_1418 (size=53) 2024-11-09T10:17:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-09T10:17:34,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:17:34,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9156a76309f41c5107dd18ab6eb49675, disabling compactions & flushes 2024-11-09T10:17:34,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:34,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:34,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. after waiting 0 ms 2024-11-09T10:17:34,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:34,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:34,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9156a76309f41c5107dd18ab6eb49675: Waiting for close lock at 1731147454982Disabling compacts and flushes for region at 1731147454982Disabling writes for close at 1731147454983 (+1 ms)Writing region close event to WAL at 1731147454983Closed at 1731147454983 2024-11-09T10:17:34,985 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T10:17:34,986 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731147454986"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731147454986"}]},"ts":"1731147454986"} 2024-11-09T10:17:34,989 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T10:17:34,991 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T10:17:34,991 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147454991"}]},"ts":"1731147454991"} 2024-11-09T10:17:34,993 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T10:17:34,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, ASSIGN}] 2024-11-09T10:17:34,995 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, ASSIGN 2024-11-09T10:17:34,996 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, ASSIGN; state=OFFLINE, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=false 2024-11-09T10:17:35,147 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=9156a76309f41c5107dd18ab6eb49675, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:35,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, ASSIGN because future has completed 2024-11-09T10:17:35,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:17:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-09T10:17:35,308 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:35,309 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:17:35,309 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,309 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:17:35,309 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,310 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,311 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,312 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:35,313 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9156a76309f41c5107dd18ab6eb49675 columnFamilyName A 2024-11-09T10:17:35,313 DEBUG [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:35,313 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.HStore(327): Store=9156a76309f41c5107dd18ab6eb49675/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:35,314 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,315 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:35,315 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9156a76309f41c5107dd18ab6eb49675 columnFamilyName B 2024-11-09T10:17:35,315 DEBUG [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:35,316 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.HStore(327): Store=9156a76309f41c5107dd18ab6eb49675/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:35,316 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,317 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:17:35,317 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9156a76309f41c5107dd18ab6eb49675 columnFamilyName C 2024-11-09T10:17:35,317 DEBUG [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:17:35,317 INFO [StoreOpener-9156a76309f41c5107dd18ab6eb49675-1 {}] regionserver.HStore(327): Store=9156a76309f41c5107dd18ab6eb49675/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:17:35,317 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:35,317 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,318 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,318 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,319 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,319 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,319 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:17:35,320 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,322 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:17:35,322 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 9156a76309f41c5107dd18ab6eb49675; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71818932, jitterRate=0.07018548250198364}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:17:35,322 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:35,322 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 9156a76309f41c5107dd18ab6eb49675: Running coprocessor pre-open hook at 1731147455310Writing region info on filesystem at 1731147455310Initializing all the Stores at 1731147455310Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147455311 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147455311Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147455311Cleaning up temporary data from old regions at 1731147455319 (+8 ms)Running coprocessor post-open hooks at 1731147455322 (+3 ms)Region opened successfully at 1731147455322 2024-11-09T10:17:35,323 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., pid=118, masterSystemTime=1731147455305 2024-11-09T10:17:35,324 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:35,325 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:35,325 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=9156a76309f41c5107dd18ab6eb49675, regionState=OPEN, openSeqNum=2, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:35,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:17:35,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-09T10:17:35,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 in 176 msec 2024-11-09T10:17:35,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-09T10:17:35,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, ASSIGN in 334 msec 2024-11-09T10:17:35,330 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T10:17:35,330 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147455330"}]},"ts":"1731147455330"} 2024-11-09T10:17:35,331 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T10:17:35,332 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T10:17:35,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-09T10:17:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-09T10:17:36,320 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:36,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460158be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,322 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266dba88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,323 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c99729d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,324 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,326 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T10:17:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e067b2c2c7b3,42133,1731147327850 2024-11-09T10:17:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@cb40c04 2024-11-09T10:17:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T10:17:36,328 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T10:17:36,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,330 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae9d428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fdd2bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d31498a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,333 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4743602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,335 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4216f56a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50286854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,338 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57de05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bac3399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f0c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,341 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171cbfb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e7663cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d01aaa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,346 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6229f747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,348 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3411d9e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,350 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b580a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,351 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,352 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c09843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,353 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74737e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16632acc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,357 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7caedf4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0bbd77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:17:36,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:17:36,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:17:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:17:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:17:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421cc3ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:17:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:17:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:36,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:17:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@712d9b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:17:36,361 DEBUG [Thread-1910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,361 DEBUG [Thread-1908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,361 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,362 DEBUG [Thread-1893 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,362 DEBUG [Thread-1895 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,362 DEBUG [Thread-1902 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-09T10:17:36,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T10:17:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,368 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,369 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:36,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:36,372 DEBUG [Thread-1906 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,372 DEBUG [Thread-1891 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,373 DEBUG [Thread-1897 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,380 DEBUG [Thread-1904 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:17:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:36,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:17:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:17:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:17:36,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:36,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:36,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:36,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:36,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:36,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:36,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:17:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2] 2024-11-09T10:17:36,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a08a6b1b50be4c859c36b4cb7070a999 is 50, key is test_row_0/A:col10/1731147456377/Put/seqid=0 2024-11-09T10:17:36,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742243_1419 (size=12001) 2024-11-09T10:17:36,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a08a6b1b50be4c859c36b4cb7070a999 2024-11-09T10:17:36,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/5af913e8b99c4337a39748512dd75227 is 50, key is test_row_0/B:col10/1731147456377/Put/seqid=0 2024-11-09T10:17:36,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742244_1420 (size=12001) 2024-11-09T10:17:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T10:17:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147516465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147456435, max timestamp=1731147456478Num uniques 859; 2024-11-09T10:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147516466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147456435, max timestamp=1731147456481Num uniques 859; 2024-11-09T10:17:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147516465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:36,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147456435, max timestamp=1731147456481Num uniques 859; 2024-11-09T10:17:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:36,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:17:36,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T10:17:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:36,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147516568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147516571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147516589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147516598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147516598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T10:17:36,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T10:17:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:36,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147516679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147516689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147516809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147516819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147516819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T10:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:36,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:36,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/5af913e8b99c4337a39748512dd75227 2024-11-09T10:17:36,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/5567230c2ed249ffae8b1ab24e0c5f5a is 50, key is test_row_0/C:col10/1731147456377/Put/seqid=0 2024-11-09T10:17:36,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742245_1421 (size=12001) 2024-11-09T10:17:36,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/5567230c2ed249ffae8b1ab24e0c5f5a 2024-11-09T10:17:36,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a08a6b1b50be4c859c36b4cb7070a999 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a08a6b1b50be4c859c36b4cb7070a999 2024-11-09T10:17:36,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a08a6b1b50be4c859c36b4cb7070a999, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T10:17:36,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/5af913e8b99c4337a39748512dd75227 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5af913e8b99c4337a39748512dd75227 2024-11-09T10:17:36,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147516899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147516899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:36,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5af913e8b99c4337a39748512dd75227, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T10:17:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/5567230c2ed249ffae8b1ab24e0c5f5a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/5567230c2ed249ffae8b1ab24e0c5f5a 2024-11-09T10:17:36,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/5567230c2ed249ffae8b1ab24e0c5f5a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T10:17:36,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9156a76309f41c5107dd18ab6eb49675 in 550ms, sequenceid=12, compaction requested=false 2024-11-09T10:17:36,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:36,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T10:17:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:36,988 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-09T10:17:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T10:17:37,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/fab6bb4f774f47a38c0783433ecd73f8 is 50, key is test_row_0/A:col10/1731147456553/Put/seqid=0 2024-11-09T10:17:37,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742246_1422 (size=12001) 2024-11-09T10:17:37,017 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/fab6bb4f774f47a38c0783433ecd73f8 2024-11-09T10:17:37,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/6db2f888197c4ef785f6c81dc56e9945 is 50, key is test_row_0/B:col10/1731147456553/Put/seqid=0 2024-11-09T10:17:37,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742247_1423 (size=12001) 2024-11-09T10:17:37,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/6db2f888197c4ef785f6c81dc56e9945 2024-11-09T10:17:37,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/0783b322074e4cfeba4cce445dae4260 is 50, key is test_row_0/C:col10/1731147456553/Put/seqid=0 2024-11-09T10:17:37,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742248_1424 (size=12001) 2024-11-09T10:17:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:37,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147517147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147517149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147517150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147517218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147517218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147517259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147517271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147517278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,482 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/0783b322074e4cfeba4cce445dae4260 2024-11-09T10:17:37,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/fab6bb4f774f47a38c0783433ecd73f8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/fab6bb4f774f47a38c0783433ecd73f8 2024-11-09T10:17:37,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147517479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,489 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/fab6bb4f774f47a38c0783433ecd73f8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T10:17:37,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/6db2f888197c4ef785f6c81dc56e9945 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/6db2f888197c4ef785f6c81dc56e9945 2024-11-09T10:17:37,493 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/6db2f888197c4ef785f6c81dc56e9945, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T10:17:37,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/0783b322074e4cfeba4cce445dae4260 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/0783b322074e4cfeba4cce445dae4260 2024-11-09T10:17:37,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147517488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,497 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/0783b322074e4cfeba4cce445dae4260, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T10:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T10:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,501 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 9156a76309f41c5107dd18ab6eb49675 in 514ms, sequenceid=48, compaction requested=false 2024-11-09T10:17:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-09T10:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-09T10:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:37,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T10:17:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:37,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-09T10:17:37,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1340 sec 2024-11-09T10:17:37,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/6ed50ac02efd449aa0a3ad63d87a6268 is 50, key is test_row_0/A:col10/1731147457502/Put/seqid=0 2024-11-09T10:17:37,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1510 sec 2024-11-09T10:17:37,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742249_1425 (size=14341) 2024-11-09T10:17:37,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/6ed50ac02efd449aa0a3ad63d87a6268 2024-11-09T10:17:37,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/84881c4a7967413db058b4c552d4ffea is 50, key is test_row_0/B:col10/1731147457502/Put/seqid=0 2024-11-09T10:17:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742250_1426 (size=12001) 2024-11-09T10:17:37,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147517639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147517738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147517738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147517759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147517799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147517808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/84881c4a7967413db058b4c552d4ffea 2024-11-09T10:17:37,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e29fc00bcae647bba229f8526700b37c is 50, key is test_row_0/C:col10/1731147457502/Put/seqid=0 2024-11-09T10:17:37,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147517969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:37,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742251_1427 (size=12001) 2024-11-09T10:17:37,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e29fc00bcae647bba229f8526700b37c 2024-11-09T10:17:38,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/6ed50ac02efd449aa0a3ad63d87a6268 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6ed50ac02efd449aa0a3ad63d87a6268 2024-11-09T10:17:38,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6ed50ac02efd449aa0a3ad63d87a6268, entries=200, sequenceid=60, filesize=14.0 K 2024-11-09T10:17:38,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/84881c4a7967413db058b4c552d4ffea as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/84881c4a7967413db058b4c552d4ffea 2024-11-09T10:17:38,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/84881c4a7967413db058b4c552d4ffea, entries=150, sequenceid=60, filesize=11.7 K 2024-11-09T10:17:38,015 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T10:17:38,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e29fc00bcae647bba229f8526700b37c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e29fc00bcae647bba229f8526700b37c 2024-11-09T10:17:38,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e29fc00bcae647bba229f8526700b37c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-09T10:17:38,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9156a76309f41c5107dd18ab6eb49675 in 528ms, sequenceid=60, compaction requested=true 2024-11-09T10:17:38,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:38,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:38,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:38,031 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:38,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:38,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:38,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:38,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T10:17:38,031 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:38,034 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:38,034 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:38,034 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:38,034 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:38,034 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,034 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,034 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5af913e8b99c4337a39748512dd75227, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/6db2f888197c4ef785f6c81dc56e9945, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/84881c4a7967413db058b4c552d4ffea] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=35.2 K 2024-11-09T10:17:38,034 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a08a6b1b50be4c859c36b4cb7070a999, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/fab6bb4f774f47a38c0783433ecd73f8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6ed50ac02efd449aa0a3ad63d87a6268] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=37.4 K 2024-11-09T10:17:38,036 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08a6b1b50be4c859c36b4cb7070a999, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731147456372 2024-11-09T10:17:38,036 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af913e8b99c4337a39748512dd75227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731147456372 2024-11-09T10:17:38,036 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting fab6bb4f774f47a38c0783433ecd73f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731147456553 2024-11-09T10:17:38,036 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 6db2f888197c4ef785f6c81dc56e9945, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731147456553 2024-11-09T10:17:38,036 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ed50ac02efd449aa0a3ad63d87a6268, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147457136 2024-11-09T10:17:38,037 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 84881c4a7967413db058b4c552d4ffea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147457136 2024-11-09T10:17:38,046 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:38,047 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/03c25664ddb747568f3775698bba0049 is 50, key is test_row_0/B:col10/1731147457502/Put/seqid=0 2024-11-09T10:17:38,059 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:38,060 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a3d34886d52442c58d9275b78c011e9b is 50, key is test_row_0/A:col10/1731147457502/Put/seqid=0 2024-11-09T10:17:38,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742252_1428 (size=12104) 2024-11-09T10:17:38,075 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/03c25664ddb747568f3775698bba0049 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/03c25664ddb747568f3775698bba0049 2024-11-09T10:17:38,086 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into 03c25664ddb747568f3775698bba0049(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:38,086 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:38,086 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=13, startTime=1731147458031; duration=0sec 2024-11-09T10:17:38,087 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:38,087 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:38,087 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:38,090 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:38,091 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:38,091 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,091 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/5567230c2ed249ffae8b1ab24e0c5f5a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/0783b322074e4cfeba4cce445dae4260, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e29fc00bcae647bba229f8526700b37c] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=35.2 K 2024-11-09T10:17:38,091 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5567230c2ed249ffae8b1ab24e0c5f5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731147456372 2024-11-09T10:17:38,091 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0783b322074e4cfeba4cce445dae4260, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731147456553 2024-11-09T10:17:38,092 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e29fc00bcae647bba229f8526700b37c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147457136 2024-11-09T10:17:38,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742253_1429 (size=12104) 2024-11-09T10:17:38,100 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:38,101 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/2aed0688a3d8418abff9c7b47b9bccee is 50, key is test_row_0/C:col10/1731147457502/Put/seqid=0 2024-11-09T10:17:38,118 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a3d34886d52442c58d9275b78c011e9b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d34886d52442c58d9275b78c011e9b 2024-11-09T10:17:38,127 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into a3d34886d52442c58d9275b78c011e9b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:38,127 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:38,127 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=13, startTime=1731147458031; duration=0sec 2024-11-09T10:17:38,127 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:38,127 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:38,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742254_1430 (size=12104) 2024-11-09T10:17:38,135 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/2aed0688a3d8418abff9c7b47b9bccee as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/2aed0688a3d8418abff9c7b47b9bccee 2024-11-09T10:17:38,141 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into 2aed0688a3d8418abff9c7b47b9bccee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:38,141 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:38,141 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=13, startTime=1731147458031; duration=0sec 2024-11-09T10:17:38,141 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:38,141 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:38,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T10:17:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:38,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/08db3538eb1b471f88fdfd0762ffbc7f is 50, key is test_row_0/A:col10/1731147457637/Put/seqid=0 2024-11-09T10:17:38,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742255_1431 (size=14341) 2024-11-09T10:17:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147518328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147518329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147518329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147518439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147518449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T10:17:38,509 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:38,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-09T10:17:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T10:17:38,510 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:38,511 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:38,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T10:17:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147518649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T10:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147518659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/08db3538eb1b471f88fdfd0762ffbc7f 2024-11-09T10:17:38,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/52aea315348c4b6c90856985a9c59a38 is 50, key is test_row_0/B:col10/1731147457637/Put/seqid=0 2024-11-09T10:17:38,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742256_1432 (size=12001) 2024-11-09T10:17:38,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147518748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,757 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147518759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,765 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T10:17:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T10:17:38,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147518958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:38,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T10:17:38,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:38,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:38,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:38,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147518979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:39,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/52aea315348c4b6c90856985a9c59a38 2024-11-09T10:17:39,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T10:17:39,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:39,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:39,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:39,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/18dfb3aad3dc4c0f9adaa2c72de70157 is 50, key is test_row_0/C:col10/1731147457637/Put/seqid=0 2024-11-09T10:17:39,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742257_1433 (size=12001) 2024-11-09T10:17:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T10:17:39,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T10:17:39,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:39,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:39,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147519349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:39,358 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:39,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T10:17:39,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:39,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:39,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:39,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:39,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147519479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:39,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147519489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:39,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/18dfb3aad3dc4c0f9adaa2c72de70157 2024-11-09T10:17:39,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/08db3538eb1b471f88fdfd0762ffbc7f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/08db3538eb1b471f88fdfd0762ffbc7f 2024-11-09T10:17:39,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/08db3538eb1b471f88fdfd0762ffbc7f, entries=200, sequenceid=84, filesize=14.0 K 2024-11-09T10:17:39,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/52aea315348c4b6c90856985a9c59a38 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/52aea315348c4b6c90856985a9c59a38 2024-11-09T10:17:39,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/52aea315348c4b6c90856985a9c59a38, entries=150, sequenceid=84, filesize=11.7 K 2024-11-09T10:17:39,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/18dfb3aad3dc4c0f9adaa2c72de70157 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/18dfb3aad3dc4c0f9adaa2c72de70157 2024-11-09T10:17:39,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/18dfb3aad3dc4c0f9adaa2c72de70157, entries=150, sequenceid=84, filesize=11.7 K 2024-11-09T10:17:39,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 9156a76309f41c5107dd18ab6eb49675 in 1264ms, sequenceid=84, compaction requested=false 2024-11-09T10:17:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:39,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T10:17:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:39,583 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:17:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/4e4616ed6bdd45cf93a5621f270a6531 is 50, key is test_row_0/A:col10/1731147458314/Put/seqid=0 2024-11-09T10:17:39,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742258_1434 (size=12001) 2024-11-09T10:17:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T10:17:40,017 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/4e4616ed6bdd45cf93a5621f270a6531 2024-11-09T10:17:40,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/13f5c84a1ea243908a5894ecdd313e36 is 50, key is test_row_0/B:col10/1731147458314/Put/seqid=0 2024-11-09T10:17:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742259_1435 (size=12001) 2024-11-09T10:17:40,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/13f5c84a1ea243908a5894ecdd313e36 2024-11-09T10:17:40,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/ed7392980dfc4f9c90515b4de1a03f4d is 50, key is test_row_0/C:col10/1731147458314/Put/seqid=0 2024-11-09T10:17:40,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742260_1436 (size=12001) 2024-11-09T10:17:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:40,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:40,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147460527, max timestamp=1731147460573Num uniques 859; 2024-11-09T10:17:40,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:40,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147460527, max timestamp=1731147460573Num uniques 859; 2024-11-09T10:17:40,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:40,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147460527, max timestamp=1731147460573Num uniques 859; 2024-11-09T10:17:40,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T10:17:40,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147520651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:40,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147520666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:40,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147520769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:40,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147520769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:40,783 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147520778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:40,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147520778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:40,793 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:40,835 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/ed7392980dfc4f9c90515b4de1a03f4d 2024-11-09T10:17:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/4e4616ed6bdd45cf93a5621f270a6531 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/4e4616ed6bdd45cf93a5621f270a6531 2024-11-09T10:17:40,847 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/4e4616ed6bdd45cf93a5621f270a6531, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T10:17:40,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/13f5c84a1ea243908a5894ecdd313e36 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/13f5c84a1ea243908a5894ecdd313e36 2024-11-09T10:17:40,852 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/13f5c84a1ea243908a5894ecdd313e36, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T10:17:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/ed7392980dfc4f9c90515b4de1a03f4d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed7392980dfc4f9c90515b4de1a03f4d 2024-11-09T10:17:40,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed7392980dfc4f9c90515b4de1a03f4d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T10:17:40,858 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9156a76309f41c5107dd18ab6eb49675 in 1275ms, sequenceid=95, compaction requested=true 2024-11-09T10:17:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-09T10:17:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-09T10:17:40,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-09T10:17:40,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-11-09T10:17:40,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.3530 sec 2024-11-09T10:17:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:41,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T10:17:41,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:41,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:41,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:41,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:41,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:41,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:41,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/016c617068234211a10e99ac953ef100 is 50, key is test_row_0/A:col10/1731147461000/Put/seqid=0 2024-11-09T10:17:41,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742261_1437 (size=14441) 2024-11-09T10:17:41,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/016c617068234211a10e99ac953ef100 2024-11-09T10:17:41,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/bae2d23fe07844de8ae82ee2f8042c78 is 50, key is test_row_0/B:col10/1731147461000/Put/seqid=0 2024-11-09T10:17:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147521057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147521061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742262_1438 (size=12101) 2024-11-09T10:17:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147521179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147521179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147521379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,386 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147521399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147521399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/bae2d23fe07844de8ae82ee2f8042c78 2024-11-09T10:17:41,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/ed06b277c216454fb3058467bf15444f is 50, key is test_row_0/C:col10/1731147461000/Put/seqid=0 2024-11-09T10:17:41,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742263_1439 (size=12101) 2024-11-09T10:17:41,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147521718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147521719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:41,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/ed06b277c216454fb3058467bf15444f 2024-11-09T10:17:41,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/016c617068234211a10e99ac953ef100 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/016c617068234211a10e99ac953ef100 2024-11-09T10:17:41,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/016c617068234211a10e99ac953ef100, entries=200, sequenceid=131, filesize=14.1 K 2024-11-09T10:17:41,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/bae2d23fe07844de8ae82ee2f8042c78 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/bae2d23fe07844de8ae82ee2f8042c78 2024-11-09T10:17:41,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/bae2d23fe07844de8ae82ee2f8042c78, entries=150, sequenceid=131, filesize=11.8 K 2024-11-09T10:17:41,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/ed06b277c216454fb3058467bf15444f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed06b277c216454fb3058467bf15444f 2024-11-09T10:17:41,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed06b277c216454fb3058467bf15444f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-09T10:17:41,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 9156a76309f41c5107dd18ab6eb49675 in 935ms, sequenceid=131, compaction requested=true 2024-11-09T10:17:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:41,937 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:41,937 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:41,938 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:41,938 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:41,938 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:41,938 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/03c25664ddb747568f3775698bba0049, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/52aea315348c4b6c90856985a9c59a38, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/13f5c84a1ea243908a5894ecdd313e36, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/bae2d23fe07844de8ae82ee2f8042c78] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=47.1 K 2024-11-09T10:17:41,939 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52887 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:41,939 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:41,939 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:41,939 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d34886d52442c58d9275b78c011e9b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/08db3538eb1b471f88fdfd0762ffbc7f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/4e4616ed6bdd45cf93a5621f270a6531, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/016c617068234211a10e99ac953ef100] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=51.6 K 2024-11-09T10:17:41,939 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d34886d52442c58d9275b78c011e9b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147457136 2024-11-09T10:17:41,939 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c25664ddb747568f3775698bba0049, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147457136 2024-11-09T10:17:41,940 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08db3538eb1b471f88fdfd0762ffbc7f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731147457606 2024-11-09T10:17:41,940 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 52aea315348c4b6c90856985a9c59a38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731147457630 2024-11-09T10:17:41,940 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4616ed6bdd45cf93a5621f270a6531, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731147458314 2024-11-09T10:17:41,940 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 13f5c84a1ea243908a5894ecdd313e36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731147458314 2024-11-09T10:17:41,940 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 016c617068234211a10e99ac953ef100, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147460627 2024-11-09T10:17:41,940 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bae2d23fe07844de8ae82ee2f8042c78, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147460627 2024-11-09T10:17:41,961 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:41,961 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:41,962 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a3d2c5eaa4074adb99288ae0ad24cdd6 is 50, key is test_row_0/A:col10/1731147461000/Put/seqid=0 2024-11-09T10:17:41,962 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/7429090771a84248a8abc0a677cd1e6f is 50, key is test_row_0/B:col10/1731147461000/Put/seqid=0 2024-11-09T10:17:41,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742264_1440 (size=12341) 2024-11-09T10:17:41,981 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a3d2c5eaa4074adb99288ae0ad24cdd6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d2c5eaa4074adb99288ae0ad24cdd6 2024-11-09T10:17:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742265_1441 (size=12341) 2024-11-09T10:17:41,990 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into a3d2c5eaa4074adb99288ae0ad24cdd6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:41,990 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:41,990 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=12, startTime=1731147461936; duration=0sec 2024-11-09T10:17:41,990 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:41,990 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:41,990 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:41,993 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:41,993 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:41,993 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:41,993 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/2aed0688a3d8418abff9c7b47b9bccee, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/18dfb3aad3dc4c0f9adaa2c72de70157, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed7392980dfc4f9c90515b4de1a03f4d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed06b277c216454fb3058467bf15444f] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=47.1 K 2024-11-09T10:17:41,995 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aed0688a3d8418abff9c7b47b9bccee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731147457136 2024-11-09T10:17:41,996 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18dfb3aad3dc4c0f9adaa2c72de70157, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731147457630 2024-11-09T10:17:41,997 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed7392980dfc4f9c90515b4de1a03f4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731147458314 2024-11-09T10:17:41,997 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed06b277c216454fb3058467bf15444f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147460627 2024-11-09T10:17:42,000 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/7429090771a84248a8abc0a677cd1e6f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7429090771a84248a8abc0a677cd1e6f 2024-11-09T10:17:42,007 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into 7429090771a84248a8abc0a677cd1e6f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:42,007 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:42,007 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=12, startTime=1731147461937; duration=0sec 2024-11-09T10:17:42,007 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:42,007 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:42,011 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:42,011 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/1e86d6ec8a2e410bb59466d6a761ab46 is 50, key is test_row_0/C:col10/1731147461000/Put/seqid=0 2024-11-09T10:17:42,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742266_1442 (size=12341) 2024-11-09T10:17:42,042 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/1e86d6ec8a2e410bb59466d6a761ab46 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/1e86d6ec8a2e410bb59466d6a761ab46 2024-11-09T10:17:42,050 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into 1e86d6ec8a2e410bb59466d6a761ab46(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:42,050 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:42,050 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=12, startTime=1731147461937; duration=0sec 2024-11-09T10:17:42,050 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:42,050 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:42,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:17:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:42,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/d7f9591ea0aa42b4bbe5e570d5fdc532 is 50, key is test_row_0/A:col10/1731147461059/Put/seqid=0 2024-11-09T10:17:42,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742267_1443 (size=14541) 2024-11-09T10:17:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147462281, max timestamp=1731147462309Num uniques 859; 2024-11-09T10:17:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147462281, max timestamp=1731147462309Num uniques 859; 2024-11-09T10:17:42,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147462281, max timestamp=1731147462309Num uniques 859; 2024-11-09T10:17:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:42,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147522390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147522396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:42,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147522508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147522509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:42,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/d7f9591ea0aa42b4bbe5e570d5fdc532 2024-11-09T10:17:42,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/0438a78617674a23b33bf85fc4fe1f8b is 50, key is test_row_0/B:col10/1731147461059/Put/seqid=0 2024-11-09T10:17:42,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742268_1444 (size=12151) 2024-11-09T10:17:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T10:17:42,669 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:42,670 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-09T10:17:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T10:17:42,671 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:42,672 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:42,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:42,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147522729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:42,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147522729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:42,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:42,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:42,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T10:17:42,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:42,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:42,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:42,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:42,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:42,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:42,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:42,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T10:17:43,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147523049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147523049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/0438a78617674a23b33bf85fc4fe1f8b 2024-11-09T10:17:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/261a561c0ae348a4a7b708426b465069 is 50, key is test_row_0/C:col10/1731147461059/Put/seqid=0 2024-11-09T10:17:43,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742269_1445 (size=12151) 2024-11-09T10:17:43,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T10:17:43,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/261a561c0ae348a4a7b708426b465069 2024-11-09T10:17:43,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/d7f9591ea0aa42b4bbe5e570d5fdc532 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/d7f9591ea0aa42b4bbe5e570d5fdc532 2024-11-09T10:17:43,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/d7f9591ea0aa42b4bbe5e570d5fdc532, entries=200, sequenceid=145, filesize=14.2 K 2024-11-09T10:17:43,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/0438a78617674a23b33bf85fc4fe1f8b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0438a78617674a23b33bf85fc4fe1f8b 2024-11-09T10:17:43,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0438a78617674a23b33bf85fc4fe1f8b, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T10:17:43,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/261a561c0ae348a4a7b708426b465069 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/261a561c0ae348a4a7b708426b465069 2024-11-09T10:17:43,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/261a561c0ae348a4a7b708426b465069, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T10:17:43,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9156a76309f41c5107dd18ab6eb49675 in 1261ms, sequenceid=145, compaction requested=false 2024-11-09T10:17:43,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:43,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T10:17:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:43,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/c917b8a5c7804f14a926457afa4d2f9e is 50, key is test_row_0/A:col10/1731147462395/Put/seqid=0 2024-11-09T10:17:43,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742270_1446 (size=14541) 2024-11-09T10:17:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147523607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147523608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147523718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147523729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T10:17:43,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:43,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147523939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147523939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:43,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/c917b8a5c7804f14a926457afa4d2f9e 2024-11-09T10:17:43,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/4809fabdab4843a095bb3bc1fe37001d is 50, key is test_row_0/B:col10/1731147462395/Put/seqid=0 2024-11-09T10:17:43,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742271_1447 (size=12151) 2024-11-09T10:17:43,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/4809fabdab4843a095bb3bc1fe37001d 2024-11-09T10:17:44,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/df050acdd0c74b5493369353f112a449 is 50, key is test_row_0/C:col10/1731147462395/Put/seqid=0 2024-11-09T10:17:44,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742272_1448 (size=12151) 2024-11-09T10:17:44,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147524248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147524248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:44,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:44,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/df050acdd0c74b5493369353f112a449 2024-11-09T10:17:44,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/c917b8a5c7804f14a926457afa4d2f9e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c917b8a5c7804f14a926457afa4d2f9e 2024-11-09T10:17:44,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c917b8a5c7804f14a926457afa4d2f9e, entries=200, sequenceid=182, filesize=14.2 K 2024-11-09T10:17:44,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/4809fabdab4843a095bb3bc1fe37001d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/4809fabdab4843a095bb3bc1fe37001d 2024-11-09T10:17:44,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/4809fabdab4843a095bb3bc1fe37001d, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T10:17:44,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/df050acdd0c74b5493369353f112a449 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/df050acdd0c74b5493369353f112a449 2024-11-09T10:17:44,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/df050acdd0c74b5493369353f112a449, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T10:17:44,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 9156a76309f41c5107dd18ab6eb49675 in 851ms, sequenceid=182, compaction requested=true 2024-11-09T10:17:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:44,428 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:44,428 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41423 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:44,429 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,429 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,429 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d2c5eaa4074adb99288ae0ad24cdd6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/d7f9591ea0aa42b4bbe5e570d5fdc532, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c917b8a5c7804f14a926457afa4d2f9e] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=40.5 K 2024-11-09T10:17:44,429 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7429090771a84248a8abc0a677cd1e6f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0438a78617674a23b33bf85fc4fe1f8b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/4809fabdab4843a095bb3bc1fe37001d] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=35.8 K 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d2c5eaa4074adb99288ae0ad24cdd6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147460627 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7429090771a84248a8abc0a677cd1e6f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147460627 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7f9591ea0aa42b4bbe5e570d5fdc532, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731147461039 2024-11-09T10:17:44,429 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0438a78617674a23b33bf85fc4fe1f8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731147461039 2024-11-09T10:17:44,430 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c917b8a5c7804f14a926457afa4d2f9e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147462376 2024-11-09T10:17:44,430 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4809fabdab4843a095bb3bc1fe37001d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147462376 2024-11-09T10:17:44,436 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:44,437 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/5201bcf9d3cc48b9a66c1d4c958cecdf is 50, key is test_row_0/A:col10/1731147462395/Put/seqid=0 2024-11-09T10:17:44,462 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:44,463 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/9a3040d3142b4b748caee0073c33a2e0 is 50, key is test_row_0/B:col10/1731147462395/Put/seqid=0 2024-11-09T10:17:44,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742273_1449 (size=12493) 2024-11-09T10:17:44,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742274_1450 (size=12493) 2024-11-09T10:17:44,476 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/5201bcf9d3cc48b9a66c1d4c958cecdf as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/5201bcf9d3cc48b9a66c1d4c958cecdf 2024-11-09T10:17:44,483 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into 5201bcf9d3cc48b9a66c1d4c958cecdf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:44,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:44,483 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=13, startTime=1731147464428; duration=0sec 2024-11-09T10:17:44,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:44,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:44,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:44,489 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/9a3040d3142b4b748caee0073c33a2e0 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9a3040d3142b4b748caee0073c33a2e0 2024-11-09T10:17:44,492 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:44,492 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:44,492 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,493 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/1e86d6ec8a2e410bb59466d6a761ab46, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/261a561c0ae348a4a7b708426b465069, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/df050acdd0c74b5493369353f112a449] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=35.8 K 2024-11-09T10:17:44,493 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e86d6ec8a2e410bb59466d6a761ab46, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147460627 2024-11-09T10:17:44,494 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261a561c0ae348a4a7b708426b465069, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731147461039 2024-11-09T10:17:44,495 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting df050acdd0c74b5493369353f112a449, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147462376 2024-11-09T10:17:44,496 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into 9a3040d3142b4b748caee0073c33a2e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:44,496 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:44,496 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=13, startTime=1731147464428; duration=0sec 2024-11-09T10:17:44,496 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:44,496 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:44,502 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:44,503 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/19738e0a42c8426facdf9bebeb684667 is 50, key is test_row_0/C:col10/1731147462395/Put/seqid=0 2024-11-09T10:17:44,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T10:17:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:44,507 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-09T10:17:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/692b32514ff744eab195937767403024 is 50, key is test_row_0/A:col10/1731147463591/Put/seqid=0 2024-11-09T10:17:44,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742275_1451 (size=12493) 2024-11-09T10:17:44,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742276_1452 (size=12151) 2024-11-09T10:17:44,520 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/19738e0a42c8426facdf9bebeb684667 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/19738e0a42c8426facdf9bebeb684667 2024-11-09T10:17:44,524 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into 19738e0a42c8426facdf9bebeb684667(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:44,524 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:44,524 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=13, startTime=1731147464428; duration=0sec 2024-11-09T10:17:44,524 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:44,524 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:44,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:44,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147464799, max timestamp=1731147464816Num uniques 859; 2024-11-09T10:17:44,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147464799, max timestamp=1731147464816Num uniques 859; 2024-11-09T10:17:44,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147464799, max timestamp=1731147464816Num uniques 859; 2024-11-09T10:17:44,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:17:44,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:17:44,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:17:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T10:17:44,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147524885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147524886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:44,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147524887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:44,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147524888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:44,919 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/692b32514ff744eab195937767403024 2024-11-09T10:17:44,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/b56d5d0fab1f453c948cce7074a9fa03 is 50, key is test_row_0/B:col10/1731147463591/Put/seqid=0 2024-11-09T10:17:44,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742277_1453 (size=12151) 2024-11-09T10:17:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147524999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147525009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147525009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147525009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147525208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147525219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147525219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147525219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,328 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/b56d5d0fab1f453c948cce7074a9fa03 2024-11-09T10:17:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e97f5a13d481482da0c3600d64490e5d is 50, key is test_row_0/C:col10/1731147463591/Put/seqid=0 2024-11-09T10:17:45,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742278_1454 (size=12151) 2024-11-09T10:17:45,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147525398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,403 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:45,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147525518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147525528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147525528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147525528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:45,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e97f5a13d481482da0c3600d64490e5d 2024-11-09T10:17:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/692b32514ff744eab195937767403024 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/692b32514ff744eab195937767403024 2024-11-09T10:17:45,754 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/692b32514ff744eab195937767403024, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T10:17:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/b56d5d0fab1f453c948cce7074a9fa03 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b56d5d0fab1f453c948cce7074a9fa03 2024-11-09T10:17:45,757 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b56d5d0fab1f453c948cce7074a9fa03, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T10:17:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e97f5a13d481482da0c3600d64490e5d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e97f5a13d481482da0c3600d64490e5d 2024-11-09T10:17:45,763 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e97f5a13d481482da0c3600d64490e5d, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T10:17:45,764 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 9156a76309f41c5107dd18ab6eb49675 in 1257ms, sequenceid=193, compaction requested=false 2024-11-09T10:17:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-09T10:17:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-09T10:17:45,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-09T10:17:45,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0930 sec 2024-11-09T10:17:45,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 3.0970 sec 2024-11-09T10:17:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:46,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-09T10:17:46,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:46,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:46,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:46,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:46,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:46,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:46,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/b72b1147a800481993819f42411a1b50 is 50, key is test_row_0/A:col10/1731147464817/Put/seqid=0 2024-11-09T10:17:46,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742279_1455 (size=14541) 2024-11-09T10:17:46,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/b72b1147a800481993819f42411a1b50 2024-11-09T10:17:46,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/24437dd321444763ae395bc9819fb469 is 50, key is test_row_0/B:col10/1731147464817/Put/seqid=0 2024-11-09T10:17:46,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147526048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147526050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147526050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147526050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742280_1456 (size=12151) 2024-11-09T10:17:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/24437dd321444763ae395bc9819fb469 2024-11-09T10:17:46,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/a02e1b0eee0b4e458b49713c9f3fd9f6 is 50, key is test_row_0/C:col10/1731147464817/Put/seqid=0 2024-11-09T10:17:46,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742281_1457 (size=12151) 2024-11-09T10:17:46,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/a02e1b0eee0b4e458b49713c9f3fd9f6 2024-11-09T10:17:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/b72b1147a800481993819f42411a1b50 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b72b1147a800481993819f42411a1b50 2024-11-09T10:17:46,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b72b1147a800481993819f42411a1b50, entries=200, sequenceid=231, filesize=14.2 K 2024-11-09T10:17:46,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/24437dd321444763ae395bc9819fb469 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/24437dd321444763ae395bc9819fb469 2024-11-09T10:17:46,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/24437dd321444763ae395bc9819fb469, entries=150, sequenceid=231, filesize=11.9 K 2024-11-09T10:17:46,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/a02e1b0eee0b4e458b49713c9f3fd9f6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/a02e1b0eee0b4e458b49713c9f3fd9f6 2024-11-09T10:17:46,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/a02e1b0eee0b4e458b49713c9f3fd9f6, entries=150, sequenceid=231, filesize=11.9 K 2024-11-09T10:17:46,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 9156a76309f41c5107dd18ab6eb49675 in 70ms, sequenceid=231, compaction requested=true 2024-11-09T10:17:46,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:46,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:46,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:46,106 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:46,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:46,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:46,106 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:46,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:46,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:46,106 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:46,106 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:46,106 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:46,107 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9a3040d3142b4b748caee0073c33a2e0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b56d5d0fab1f453c948cce7074a9fa03, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/24437dd321444763ae395bc9819fb469] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=35.9 K 2024-11-09T10:17:46,107 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:46,107 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:46,107 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a3040d3142b4b748caee0073c33a2e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147462376 2024-11-09T10:17:46,107 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:46,107 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/5201bcf9d3cc48b9a66c1d4c958cecdf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/692b32514ff744eab195937767403024, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b72b1147a800481993819f42411a1b50] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=38.3 K 2024-11-09T10:17:46,107 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b56d5d0fab1f453c948cce7074a9fa03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147463589 2024-11-09T10:17:46,107 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5201bcf9d3cc48b9a66c1d4c958cecdf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147462376 2024-11-09T10:17:46,108 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 24437dd321444763ae395bc9819fb469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731147464817 2024-11-09T10:17:46,108 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692b32514ff744eab195937767403024, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147463589 2024-11-09T10:17:46,108 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72b1147a800481993819f42411a1b50, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731147464817 2024-11-09T10:17:46,135 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:46,135 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a4c19e416e7f4b2ca30e4beb2e7e6a01 is 50, key is test_row_0/A:col10/1731147464817/Put/seqid=0 2024-11-09T10:17:46,142 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:46,143 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/94ea3cc2216e4e4f9d4511d86df38b5c is 50, key is test_row_0/B:col10/1731147464817/Put/seqid=0 2024-11-09T10:17:46,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742282_1458 (size=12595) 2024-11-09T10:17:46,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742283_1459 (size=12595) 2024-11-09T10:17:46,164 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/94ea3cc2216e4e4f9d4511d86df38b5c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/94ea3cc2216e4e4f9d4511d86df38b5c 2024-11-09T10:17:46,170 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into 94ea3cc2216e4e4f9d4511d86df38b5c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:46,170 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:46,170 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=13, startTime=1731147466106; duration=0sec 2024-11-09T10:17:46,170 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:46,170 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:46,170 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:46,171 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:46,171 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:46,171 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:46,171 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/19738e0a42c8426facdf9bebeb684667, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e97f5a13d481482da0c3600d64490e5d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/a02e1b0eee0b4e458b49713c9f3fd9f6] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=35.9 K 2024-11-09T10:17:46,172 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 19738e0a42c8426facdf9bebeb684667, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731147462376 2024-11-09T10:17:46,173 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e97f5a13d481482da0c3600d64490e5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731147463589 2024-11-09T10:17:46,173 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a02e1b0eee0b4e458b49713c9f3fd9f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731147464817 2024-11-09T10:17:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:46,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:17:46,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:46,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:46,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:46,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:46,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:46,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:46,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/408fe0a4fe894a9d84f4afce56f223ad is 50, key is test_row_0/A:col10/1731147466175/Put/seqid=0 2024-11-09T10:17:46,184 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:46,185 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/6ca2eed4425d46479ba9cf2d87d226e4 is 50, key is test_row_0/C:col10/1731147464817/Put/seqid=0 2024-11-09T10:17:46,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742284_1460 (size=16931) 2024-11-09T10:17:46,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742285_1461 (size=12595) 2024-11-09T10:17:46,205 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/6ca2eed4425d46479ba9cf2d87d226e4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/6ca2eed4425d46479ba9cf2d87d226e4 2024-11-09T10:17:46,209 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into 6ca2eed4425d46479ba9cf2d87d226e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:46,209 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:46,209 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=13, startTime=1731147466106; duration=0sec 2024-11-09T10:17:46,209 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:46,209 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147466197, max timestamp=1731147466231Num uniques 859; 2024-11-09T10:17:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147466197, max timestamp=1731147466231Num uniques 859; 2024-11-09T10:17:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147466197, max timestamp=1731147466231Num uniques 859; 2024-11-09T10:17:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:17:46,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147526274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147526274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147526275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147526276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147526399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147526399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147526400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147526400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,562 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/a4c19e416e7f4b2ca30e4beb2e7e6a01 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a4c19e416e7f4b2ca30e4beb2e7e6a01 2024-11-09T10:17:46,570 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into a4c19e416e7f4b2ca30e4beb2e7e6a01(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:46,570 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:46,570 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=13, startTime=1731147466105; duration=0sec 2024-11-09T10:17:46,570 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:46,570 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:46,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/408fe0a4fe894a9d84f4afce56f223ad 2024-11-09T10:17:46,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/e691d39843e84d8c9edfc72bdfb8c9bc is 50, key is test_row_0/B:col10/1731147466175/Put/seqid=0 2024-11-09T10:17:46,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742286_1462 (size=12151) 2024-11-09T10:17:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147526609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147526609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147526609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147526609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T10:17:46,849 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:46,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-09T10:17:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T10:17:46,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:46,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:46,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147526929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147526929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147526929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147526929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T10:17:47,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T10:17:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:47,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:47,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:47,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/e691d39843e84d8c9edfc72bdfb8c9bc 2024-11-09T10:17:47,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/c832de8ad1ff4242b88f066119cd38d4 is 50, key is test_row_0/C:col10/1731147466175/Put/seqid=0 2024-11-09T10:17:47,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742287_1463 (size=12151) 2024-11-09T10:17:47,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/c832de8ad1ff4242b88f066119cd38d4 2024-11-09T10:17:47,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/408fe0a4fe894a9d84f4afce56f223ad as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/408fe0a4fe894a9d84f4afce56f223ad 2024-11-09T10:17:47,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/408fe0a4fe894a9d84f4afce56f223ad, entries=250, sequenceid=242, filesize=16.5 K 2024-11-09T10:17:47,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/e691d39843e84d8c9edfc72bdfb8c9bc as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/e691d39843e84d8c9edfc72bdfb8c9bc 2024-11-09T10:17:47,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/e691d39843e84d8c9edfc72bdfb8c9bc, entries=150, sequenceid=242, filesize=11.9 K 2024-11-09T10:17:47,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/c832de8ad1ff4242b88f066119cd38d4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/c832de8ad1ff4242b88f066119cd38d4 2024-11-09T10:17:47,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/c832de8ad1ff4242b88f066119cd38d4, entries=150, sequenceid=242, filesize=11.9 K 2024-11-09T10:17:47,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9156a76309f41c5107dd18ab6eb49675 in 859ms, sequenceid=242, compaction requested=false 2024-11-09T10:17:47,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:47,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T10:17:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:47,156 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T10:17:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:47,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:47,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:17:47,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/53bc0533b9de45eaa2c80b941ee9fd59 is 50, key is test_row_0/A:col10/1731147466275/Put/seqid=0 2024-11-09T10:17:47,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742288_1464 (size=12301) 2024-11-09T10:17:47,168 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/53bc0533b9de45eaa2c80b941ee9fd59 2024-11-09T10:17:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T10:17:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2475ac65fc314a2a8b1829d6aceaec6b is 50, key is test_row_0/B:col10/1731147466275/Put/seqid=0 2024-11-09T10:17:47,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742289_1465 (size=12301) 2024-11-09T10:17:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:47,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147527463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147527464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147527471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147527471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T10:17:47,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147527579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,584 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2475ac65fc314a2a8b1829d6aceaec6b 2024-11-09T10:17:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147527579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147527579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147527579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/8652acc9488c4d0294edb8ab0d5d0e5c is 50, key is test_row_0/C:col10/1731147466275/Put/seqid=0 2024-11-09T10:17:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742290_1466 (size=12301) 2024-11-09T10:17:47,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147527789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147527799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147527799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147527799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T10:17:47,994 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/8652acc9488c4d0294edb8ab0d5d0e5c 2024-11-09T10:17:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/53bc0533b9de45eaa2c80b941ee9fd59 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/53bc0533b9de45eaa2c80b941ee9fd59 2024-11-09T10:17:48,001 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/53bc0533b9de45eaa2c80b941ee9fd59, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T10:17:48,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2475ac65fc314a2a8b1829d6aceaec6b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2475ac65fc314a2a8b1829d6aceaec6b 2024-11-09T10:17:48,006 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2475ac65fc314a2a8b1829d6aceaec6b, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T10:17:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/8652acc9488c4d0294edb8ab0d5d0e5c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8652acc9488c4d0294edb8ab0d5d0e5c 2024-11-09T10:17:48,010 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8652acc9488c4d0294edb8ab0d5d0e5c, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T10:17:48,010 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 9156a76309f41c5107dd18ab6eb49675 in 854ms, sequenceid=280, compaction requested=true 2024-11-09T10:17:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-09T10:17:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-09T10:17:48,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-09T10:17:48,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1610 sec 2024-11-09T10:17:48,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.1650 sec 2024-11-09T10:17:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:48,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T10:17:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:48,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/bf0823755c634c3698e957e5ea2e2b2f is 50, key is test_row_0/A:col10/1731147467470/Put/seqid=0 2024-11-09T10:17:48,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742291_1467 (size=14741) 2024-11-09T10:17:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147528161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147528162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147528163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147528166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147528269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147528269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147528278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147528279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147528479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147528488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147528489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147528489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/bf0823755c634c3698e957e5ea2e2b2f 2024-11-09T10:17:48,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/c3973840a96343469e303779b4306563 is 50, key is test_row_0/B:col10/1731147467470/Put/seqid=0 2024-11-09T10:17:48,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742292_1468 (size=12301) 2024-11-09T10:17:48,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147528799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147528799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147528799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147528809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:48,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/c3973840a96343469e303779b4306563 2024-11-09T10:17:48,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/acd2375e73c6444199eee66ab106d6e5 is 50, key is test_row_0/C:col10/1731147467470/Put/seqid=0 2024-11-09T10:17:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742293_1469 (size=12301) 2024-11-09T10:17:48,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/acd2375e73c6444199eee66ab106d6e5 2024-11-09T10:17:48,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/bf0823755c634c3698e957e5ea2e2b2f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/bf0823755c634c3698e957e5ea2e2b2f 2024-11-09T10:17:48,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/bf0823755c634c3698e957e5ea2e2b2f, entries=200, sequenceid=292, filesize=14.4 K 2024-11-09T10:17:48,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/c3973840a96343469e303779b4306563 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c3973840a96343469e303779b4306563 2024-11-09T10:17:48,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c3973840a96343469e303779b4306563, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T10:17:48,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/acd2375e73c6444199eee66ab106d6e5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/acd2375e73c6444199eee66ab106d6e5 2024-11-09T10:17:48,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/acd2375e73c6444199eee66ab106d6e5, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T10:17:48,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9156a76309f41c5107dd18ab6eb49675 in 864ms, sequenceid=292, compaction requested=true 2024-11-09T10:17:48,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:48,978 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:48,979 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56568 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:48,979 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:48,979 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:48,979 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a4c19e416e7f4b2ca30e4beb2e7e6a01, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/408fe0a4fe894a9d84f4afce56f223ad, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/53bc0533b9de45eaa2c80b941ee9fd59, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/bf0823755c634c3698e957e5ea2e2b2f] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=55.2 K 2024-11-09T10:17:48,979 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c19e416e7f4b2ca30e4beb2e7e6a01, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731147464817 2024-11-09T10:17:48,979 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408fe0a4fe894a9d84f4afce56f223ad, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731147466046 2024-11-09T10:17:48,980 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53bc0533b9de45eaa2c80b941ee9fd59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731147466272 2024-11-09T10:17:48,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:48,980 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf0823755c634c3698e957e5ea2e2b2f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731147467455 2024-11-09T10:17:48,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:48,983 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:48,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:48,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:48,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:48,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:48,984 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:48,984 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:48,984 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:48,984 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/94ea3cc2216e4e4f9d4511d86df38b5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/e691d39843e84d8c9edfc72bdfb8c9bc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2475ac65fc314a2a8b1829d6aceaec6b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c3973840a96343469e303779b4306563] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=48.2 K 2024-11-09T10:17:48,985 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ea3cc2216e4e4f9d4511d86df38b5c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731147464817 2024-11-09T10:17:48,986 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e691d39843e84d8c9edfc72bdfb8c9bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731147466047 2024-11-09T10:17:48,986 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2475ac65fc314a2a8b1829d6aceaec6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731147466272 2024-11-09T10:17:48,986 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c3973840a96343469e303779b4306563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731147467462 2024-11-09T10:17:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T10:17:48,999 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:48,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-09T10:17:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T10:17:49,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:49,003 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:49,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:49,005 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:49,006 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/c6ab1189f7384a4895ee70d4f0598a3e is 50, key is test_row_0/A:col10/1731147467470/Put/seqid=0 2024-11-09T10:17:49,023 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:49,024 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/3a89e4ea3d594a22ae04d5969accf87b is 50, key is test_row_0/B:col10/1731147467470/Put/seqid=0 2024-11-09T10:17:49,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742294_1470 (size=12881) 2024-11-09T10:17:49,049 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/c6ab1189f7384a4895ee70d4f0598a3e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c6ab1189f7384a4895ee70d4f0598a3e 2024-11-09T10:17:49,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742295_1471 (size=12881) 2024-11-09T10:17:49,054 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into c6ab1189f7384a4895ee70d4f0598a3e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:49,054 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:49,054 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=12, startTime=1731147468978; duration=0sec 2024-11-09T10:17:49,055 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:49,055 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:49,055 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:49,055 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:49,055 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:49,055 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:49,056 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/6ca2eed4425d46479ba9cf2d87d226e4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/c832de8ad1ff4242b88f066119cd38d4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8652acc9488c4d0294edb8ab0d5d0e5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/acd2375e73c6444199eee66ab106d6e5] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=48.2 K 2024-11-09T10:17:49,056 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ca2eed4425d46479ba9cf2d87d226e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731147464817 2024-11-09T10:17:49,056 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c832de8ad1ff4242b88f066119cd38d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731147466047 2024-11-09T10:17:49,056 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8652acc9488c4d0294edb8ab0d5d0e5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731147466272 2024-11-09T10:17:49,056 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd2375e73c6444199eee66ab106d6e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731147467462 2024-11-09T10:17:49,066 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:49,066 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/db27de0e23304452b9eaebe0d43bbcc8 is 50, key is test_row_0/C:col10/1731147467470/Put/seqid=0 2024-11-09T10:17:49,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742296_1472 (size=12881) 2024-11-09T10:17:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T10:17:49,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T10:17:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:49,156 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T10:17:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:49,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/86eb8936160d47de99400b5b08616555 is 50, key is test_row_0/A:col10/1731147468161/Put/seqid=0 2024-11-09T10:17:49,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742297_1473 (size=12301) 2024-11-09T10:17:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T10:17:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:49,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147529379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147529380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147529381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147529388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,455 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/3a89e4ea3d594a22ae04d5969accf87b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/3a89e4ea3d594a22ae04d5969accf87b 2024-11-09T10:17:49,458 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into 3a89e4ea3d594a22ae04d5969accf87b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:49,458 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:49,458 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=12, startTime=1731147468983; duration=0sec 2024-11-09T10:17:49,458 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:49,458 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:49,478 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/db27de0e23304452b9eaebe0d43bbcc8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/db27de0e23304452b9eaebe0d43bbcc8 2024-11-09T10:17:49,482 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into db27de0e23304452b9eaebe0d43bbcc8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:49,482 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:49,482 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=12, startTime=1731147468984; duration=0sec 2024-11-09T10:17:49,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:49,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:49,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147529499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147529508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147529508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147529508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,568 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/86eb8936160d47de99400b5b08616555 2024-11-09T10:17:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/1fff6ec33cfb4602ae2af10b66ea5cb1 is 50, key is test_row_0/B:col10/1731147468161/Put/seqid=0 2024-11-09T10:17:49,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742298_1474 (size=12301) 2024-11-09T10:17:49,580 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/1fff6ec33cfb4602ae2af10b66ea5cb1 2024-11-09T10:17:49,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e8891c51d2744b58ad1669829ae1e9ac is 50, key is test_row_0/C:col10/1731147468161/Put/seqid=0 2024-11-09T10:17:49,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742299_1475 (size=12301) 2024-11-09T10:17:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T10:17:49,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147529709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147529719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147529719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147529719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:49,989 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e8891c51d2744b58ad1669829ae1e9ac 2024-11-09T10:17:49,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/86eb8936160d47de99400b5b08616555 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/86eb8936160d47de99400b5b08616555 2024-11-09T10:17:49,995 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/86eb8936160d47de99400b5b08616555, entries=150, sequenceid=313, filesize=12.0 K 2024-11-09T10:17:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/1fff6ec33cfb4602ae2af10b66ea5cb1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/1fff6ec33cfb4602ae2af10b66ea5cb1 2024-11-09T10:17:49,998 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/1fff6ec33cfb4602ae2af10b66ea5cb1, entries=150, sequenceid=313, filesize=12.0 K 2024-11-09T10:17:49,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/e8891c51d2744b58ad1669829ae1e9ac as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e8891c51d2744b58ad1669829ae1e9ac 2024-11-09T10:17:50,001 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e8891c51d2744b58ad1669829ae1e9ac, entries=150, sequenceid=313, filesize=12.0 K 2024-11-09T10:17:50,002 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9156a76309f41c5107dd18ab6eb49675 in 846ms, sequenceid=313, compaction requested=false 2024-11-09T10:17:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-09T10:17:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-09T10:17:50,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-09T10:17:50,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0000 sec 2024-11-09T10:17:50,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0050 sec 2024-11-09T10:17:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:50,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:17:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:50,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/e827e2b34e0f47afa4ada9f125862593 is 50, key is test_row_0/A:col10/1731147469379/Put/seqid=0 2024-11-09T10:17:50,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742300_1476 (size=14741) 2024-11-09T10:17:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147530073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147530074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147530074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147530075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T10:17:50,139 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:50,140 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-09T10:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T10:17:50,142 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:50,142 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:50,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:50,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147530200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147530200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147530200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147530202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T10:17:50,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:50,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:50,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147530409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147530410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147530410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147530410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:50,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:50,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T10:17:50,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/e827e2b34e0f47afa4ada9f125862593 2024-11-09T10:17:50,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/0d0df719780a4bcfa7ae7cb11a751496 is 50, key is test_row_0/B:col10/1731147469379/Put/seqid=0 2024-11-09T10:17:50,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742301_1477 (size=12301) 2024-11-09T10:17:50,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147530728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147530729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147530729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147530729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:50,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T10:17:50,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/0d0df719780a4bcfa7ae7cb11a751496 2024-11-09T10:17:50,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/4bc5504a7c5641b495ef13819a11dcd9 is 50, key is test_row_0/C:col10/1731147469379/Put/seqid=0 2024-11-09T10:17:50,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742302_1478 (size=12301) 2024-11-09T10:17:50,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:50,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:51,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:51,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:51,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147531239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:51,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147531239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147531249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147531249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T10:17:51,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/4bc5504a7c5641b495ef13819a11dcd9 2024-11-09T10:17:51,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/e827e2b34e0f47afa4ada9f125862593 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/e827e2b34e0f47afa4ada9f125862593 2024-11-09T10:17:51,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/e827e2b34e0f47afa4ada9f125862593, entries=200, sequenceid=328, filesize=14.4 K 2024-11-09T10:17:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/0d0df719780a4bcfa7ae7cb11a751496 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0d0df719780a4bcfa7ae7cb11a751496 2024-11-09T10:17:51,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0d0df719780a4bcfa7ae7cb11a751496, entries=150, sequenceid=328, filesize=12.0 K 2024-11-09T10:17:51,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/4bc5504a7c5641b495ef13819a11dcd9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/4bc5504a7c5641b495ef13819a11dcd9 2024-11-09T10:17:51,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/4bc5504a7c5641b495ef13819a11dcd9, entries=150, sequenceid=328, filesize=12.0 K 2024-11-09T10:17:51,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9156a76309f41c5107dd18ab6eb49675 in 1267ms, sequenceid=328, compaction requested=true 2024-11-09T10:17:51,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:51,301 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:51,301 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:51,304 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:51,304 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:51,304 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:51,304 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:51,304 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,304 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,305 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c6ab1189f7384a4895ee70d4f0598a3e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/86eb8936160d47de99400b5b08616555, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/e827e2b34e0f47afa4ada9f125862593] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=39.0 K 2024-11-09T10:17:51,305 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/3a89e4ea3d594a22ae04d5969accf87b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/1fff6ec33cfb4602ae2af10b66ea5cb1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0d0df719780a4bcfa7ae7cb11a751496] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=36.6 K 2024-11-09T10:17:51,305 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6ab1189f7384a4895ee70d4f0598a3e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731147467462 2024-11-09T10:17:51,305 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a89e4ea3d594a22ae04d5969accf87b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731147467462 2024-11-09T10:17:51,305 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86eb8936160d47de99400b5b08616555, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731147468160 2024-11-09T10:17:51,305 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fff6ec33cfb4602ae2af10b66ea5cb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731147468160 2024-11-09T10:17:51,305 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e827e2b34e0f47afa4ada9f125862593, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731147469379 2024-11-09T10:17:51,305 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0df719780a4bcfa7ae7cb11a751496, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731147469379 2024-11-09T10:17:51,314 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:51,314 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/c22b64ef60dc4fa2a8ef6d9178282270 is 50, key is test_row_0/B:col10/1731147469379/Put/seqid=0 2024-11-09T10:17:51,318 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:51,319 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/624f5839a16a4c1b888483536e37b450 is 50, key is test_row_0/A:col10/1731147469379/Put/seqid=0 2024-11-09T10:17:51,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742303_1479 (size=12983) 2024-11-09T10:17:51,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742304_1480 (size=12983) 2024-11-09T10:17:51,329 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/c22b64ef60dc4fa2a8ef6d9178282270 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c22b64ef60dc4fa2a8ef6d9178282270 2024-11-09T10:17:51,332 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into c22b64ef60dc4fa2a8ef6d9178282270(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:51,332 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:51,332 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=13, startTime=1731147471301; duration=0sec 2024-11-09T10:17:51,333 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:51,333 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:51,333 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:51,334 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:51,334 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:51,335 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,335 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/db27de0e23304452b9eaebe0d43bbcc8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e8891c51d2744b58ad1669829ae1e9ac, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/4bc5504a7c5641b495ef13819a11dcd9] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=36.6 K 2024-11-09T10:17:51,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting db27de0e23304452b9eaebe0d43bbcc8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731147467462 2024-11-09T10:17:51,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e8891c51d2744b58ad1669829ae1e9ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731147468160 2024-11-09T10:17:51,335 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bc5504a7c5641b495ef13819a11dcd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731147469379 2024-11-09T10:17:51,337 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/624f5839a16a4c1b888483536e37b450 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/624f5839a16a4c1b888483536e37b450 2024-11-09T10:17:51,343 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into 624f5839a16a4c1b888483536e37b450(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:51,343 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:51,343 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=13, startTime=1731147471300; duration=0sec 2024-11-09T10:17:51,343 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:51,343 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:51,344 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:51,345 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/46f87ca3fc434294880d889187940795 is 50, key is test_row_0/C:col10/1731147469379/Put/seqid=0 2024-11-09T10:17:51,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742305_1481 (size=12983) 2024-11-09T10:17:51,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T10:17:51,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:51,362 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:17:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:51,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/6bef92156b8d4f0ca94a8555df7408f1 is 50, key is test_row_0/A:col10/1731147470074/Put/seqid=0 2024-11-09T10:17:51,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742306_1482 (size=12301) 2024-11-09T10:17:51,753 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/46f87ca3fc434294880d889187940795 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/46f87ca3fc434294880d889187940795 2024-11-09T10:17:51,756 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into 46f87ca3fc434294880d889187940795(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:51,756 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:51,756 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=13, startTime=1731147471301; duration=0sec 2024-11-09T10:17:51,756 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:51,756 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:51,777 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/6bef92156b8d4f0ca94a8555df7408f1 2024-11-09T10:17:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/b76fd7e43acd4129a9c921bc85f7165b is 50, key is test_row_0/B:col10/1731147470074/Put/seqid=0 2024-11-09T10:17:51,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742307_1483 (size=12301) 2024-11-09T10:17:52,187 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/b76fd7e43acd4129a9c921bc85f7165b 2024-11-09T10:17:52,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/dfd1bd5f9ad8402dae8f60c5f3627285 is 50, key is test_row_0/C:col10/1731147470074/Put/seqid=0 2024-11-09T10:17:52,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742308_1484 (size=12301) 2024-11-09T10:17:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147532275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147532276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147532277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147532278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T10:17:52,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147532388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147532389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147532389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147532389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,596 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/dfd1bd5f9ad8402dae8f60c5f3627285 2024-11-09T10:17:52,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/6bef92156b8d4f0ca94a8555df7408f1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6bef92156b8d4f0ca94a8555df7408f1 2024-11-09T10:17:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147532599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147532599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147532599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147532599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,607 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6bef92156b8d4f0ca94a8555df7408f1, entries=150, sequenceid=349, filesize=12.0 K 2024-11-09T10:17:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/b76fd7e43acd4129a9c921bc85f7165b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b76fd7e43acd4129a9c921bc85f7165b 2024-11-09T10:17:52,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b76fd7e43acd4129a9c921bc85f7165b, entries=150, sequenceid=349, filesize=12.0 K 2024-11-09T10:17:52,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/dfd1bd5f9ad8402dae8f60c5f3627285 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/dfd1bd5f9ad8402dae8f60c5f3627285 2024-11-09T10:17:52,619 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/dfd1bd5f9ad8402dae8f60c5f3627285, entries=150, sequenceid=349, filesize=12.0 K 2024-11-09T10:17:52,620 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9156a76309f41c5107dd18ab6eb49675 in 1257ms, sequenceid=349, compaction requested=false 2024-11-09T10:17:52,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:52,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:52,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-09T10:17:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-09T10:17:52,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-09T10:17:52,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4790 sec 2024-11-09T10:17:52,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.4830 sec 2024-11-09T10:17:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:52,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T10:17:52,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:52,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:52,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:52,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:52,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:52,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:52,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/2548fc4a44d24e1aa8c28d5fa3f08ef4 is 50, key is test_row_0/A:col10/1731147472277/Put/seqid=0 2024-11-09T10:17:52,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742309_1485 (size=14741) 2024-11-09T10:17:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147532939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147532940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147532940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147532941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147533059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147533059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147533059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147533059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147533269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147533269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147533269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147533269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/2548fc4a44d24e1aa8c28d5fa3f08ef4 2024-11-09T10:17:53,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/7080895927ab43eeae978877866768e8 is 50, key is test_row_0/B:col10/1731147472277/Put/seqid=0 2024-11-09T10:17:53,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742310_1486 (size=12301) 2024-11-09T10:17:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147533578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147533589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147533589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147533589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:53,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/7080895927ab43eeae978877866768e8 2024-11-09T10:17:53,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/672461a8008c49918966c880b2c9c583 is 50, key is test_row_0/C:col10/1731147472277/Put/seqid=0 2024-11-09T10:17:53,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742311_1487 (size=12301) 2024-11-09T10:17:53,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/672461a8008c49918966c880b2c9c583 2024-11-09T10:17:53,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/2548fc4a44d24e1aa8c28d5fa3f08ef4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/2548fc4a44d24e1aa8c28d5fa3f08ef4 2024-11-09T10:17:53,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/2548fc4a44d24e1aa8c28d5fa3f08ef4, entries=200, sequenceid=364, filesize=14.4 K 2024-11-09T10:17:53,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/7080895927ab43eeae978877866768e8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7080895927ab43eeae978877866768e8 2024-11-09T10:17:53,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7080895927ab43eeae978877866768e8, entries=150, sequenceid=364, filesize=12.0 K 2024-11-09T10:17:53,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/672461a8008c49918966c880b2c9c583 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/672461a8008c49918966c880b2c9c583 2024-11-09T10:17:53,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/672461a8008c49918966c880b2c9c583, entries=150, sequenceid=364, filesize=12.0 K 2024-11-09T10:17:53,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9156a76309f41c5107dd18ab6eb49675 in 839ms, sequenceid=364, compaction requested=true 2024-11-09T10:17:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:53,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:53,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:53,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:53,756 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:53,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:53,756 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:53,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:53,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:53,756 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:53,756 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:53,756 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:53,756 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:53,756 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:53,756 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:53,756 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c22b64ef60dc4fa2a8ef6d9178282270, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b76fd7e43acd4129a9c921bc85f7165b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7080895927ab43eeae978877866768e8] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=36.7 K 2024-11-09T10:17:53,756 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/624f5839a16a4c1b888483536e37b450, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6bef92156b8d4f0ca94a8555df7408f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/2548fc4a44d24e1aa8c28d5fa3f08ef4] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=39.1 K 2024-11-09T10:17:53,757 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c22b64ef60dc4fa2a8ef6d9178282270, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731147469379 2024-11-09T10:17:53,757 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624f5839a16a4c1b888483536e37b450, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731147469379 2024-11-09T10:17:53,757 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b76fd7e43acd4129a9c921bc85f7165b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731147470070 2024-11-09T10:17:53,757 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bef92156b8d4f0ca94a8555df7408f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731147470070 2024-11-09T10:17:53,757 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7080895927ab43eeae978877866768e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731147472263 2024-11-09T10:17:53,757 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2548fc4a44d24e1aa8c28d5fa3f08ef4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731147472263 2024-11-09T10:17:53,765 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:53,766 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/59ae61992c534f70bad58add2b282008 is 50, key is test_row_0/A:col10/1731147472277/Put/seqid=0 2024-11-09T10:17:53,766 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:53,766 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/216723f40aea42b28629d6c5e7c1eb51 is 50, key is test_row_0/B:col10/1731147472277/Put/seqid=0 2024-11-09T10:17:53,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742312_1488 (size=13085) 2024-11-09T10:17:53,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742313_1489 (size=13085) 2024-11-09T10:17:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:54,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:17:54,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:54,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:54,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:54,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:54,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:54,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:54,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/18a8471434f14ab9bf71638b8b90b54e is 50, key is test_row_0/A:col10/1731147472939/Put/seqid=0 2024-11-09T10:17:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742314_1490 (size=14741) 2024-11-09T10:17:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147534124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147534124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147534129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147534137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,173 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/59ae61992c534f70bad58add2b282008 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/59ae61992c534f70bad58add2b282008 2024-11-09T10:17:54,177 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into 59ae61992c534f70bad58add2b282008(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:54,177 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:54,177 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=13, startTime=1731147473755; duration=0sec 2024-11-09T10:17:54,177 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:54,177 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:54,177 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:17:54,178 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:17:54,178 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:54,178 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,178 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/46f87ca3fc434294880d889187940795, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/dfd1bd5f9ad8402dae8f60c5f3627285, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/672461a8008c49918966c880b2c9c583] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=36.7 K 2024-11-09T10:17:54,178 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46f87ca3fc434294880d889187940795, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731147469379 2024-11-09T10:17:54,179 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfd1bd5f9ad8402dae8f60c5f3627285, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731147470070 2024-11-09T10:17:54,179 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 672461a8008c49918966c880b2c9c583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731147472263 2024-11-09T10:17:54,181 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/216723f40aea42b28629d6c5e7c1eb51 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/216723f40aea42b28629d6c5e7c1eb51 2024-11-09T10:17:54,184 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:54,185 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/70888836e2a942f9bdc4fa754e9a2edf is 50, key is test_row_0/C:col10/1731147472277/Put/seqid=0 2024-11-09T10:17:54,187 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into 216723f40aea42b28629d6c5e7c1eb51(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:54,187 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:54,187 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=13, startTime=1731147473756; duration=0sec 2024-11-09T10:17:54,187 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:54,187 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:54,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742315_1491 (size=13085) 2024-11-09T10:17:54,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147534238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147534238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147534238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147534249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T10:17:54,299 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:54,300 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:17:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-09T10:17:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T10:17:54,303 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:17:54,304 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:17:54,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:17:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T10:17:54,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T10:17:54,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:54,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147534458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147534459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147534459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147534459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/18a8471434f14ab9bf71638b8b90b54e 2024-11-09T10:17:54,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2cc9942230584ea19f27968fa8742b57 is 50, key is test_row_0/B:col10/1731147472939/Put/seqid=0 2024-11-09T10:17:54,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742316_1492 (size=12301) 2024-11-09T10:17:54,597 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/70888836e2a942f9bdc4fa754e9a2edf as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/70888836e2a942f9bdc4fa754e9a2edf 2024-11-09T10:17:54,601 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into 70888836e2a942f9bdc4fa754e9a2edf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:54,601 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:54,601 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=13, startTime=1731147473756; duration=0sec 2024-11-09T10:17:54,601 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:54,601 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:54,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T10:17:54,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:54,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T10:17:54,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T10:17:54,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:54,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147534769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147534771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147534771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147534778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:54,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T10:17:54,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:54,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:54,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2cc9942230584ea19f27968fa8742b57 2024-11-09T10:17:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:54,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/21d47d71ceef422783989b839e3366d5 is 50, key is test_row_0/C:col10/1731147472939/Put/seqid=0 2024-11-09T10:17:54,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742317_1493 (size=12301) 2024-11-09T10:17:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T10:17:55,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T10:17:55,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:55,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:55,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:55,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T10:17:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:55,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:17:55,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147535288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:55,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147535288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147535288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147535289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:55,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/21d47d71ceef422783989b839e3366d5 2024-11-09T10:17:55,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/18a8471434f14ab9bf71638b8b90b54e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/18a8471434f14ab9bf71638b8b90b54e 2024-11-09T10:17:55,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/18a8471434f14ab9bf71638b8b90b54e, entries=200, sequenceid=383, filesize=14.4 K 2024-11-09T10:17:55,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2cc9942230584ea19f27968fa8742b57 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2cc9942230584ea19f27968fa8742b57 2024-11-09T10:17:55,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2cc9942230584ea19f27968fa8742b57, entries=150, sequenceid=383, filesize=12.0 K 2024-11-09T10:17:55,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/21d47d71ceef422783989b839e3366d5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/21d47d71ceef422783989b839e3366d5 2024-11-09T10:17:55,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/21d47d71ceef422783989b839e3366d5, entries=150, sequenceid=383, filesize=12.0 K 2024-11-09T10:17:55,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9156a76309f41c5107dd18ab6eb49675 in 1248ms, sequenceid=383, compaction requested=false 2024-11-09T10:17:55,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:55,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T10:17:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:55,372 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:17:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:55,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/95d849836f8344fcbd5f3be5030586fe is 50, key is test_row_0/A:col10/1731147474101/Put/seqid=0 2024-11-09T10:17:55,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742318_1494 (size=12301) 2024-11-09T10:17:55,392 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/95d849836f8344fcbd5f3be5030586fe 2024-11-09T10:17:55,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2ec57f520b0842d8a5b815ce741bf02c is 50, key is test_row_0/B:col10/1731147474101/Put/seqid=0 2024-11-09T10:17:55,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742319_1495 (size=12301) 2024-11-09T10:17:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. as already flushing 2024-11-09T10:17:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T10:17:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147535513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147535628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:55,802 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2ec57f520b0842d8a5b815ce741bf02c 2024-11-09T10:17:55,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/68b68054d5044f5a88e061cf69fc42cf is 50, key is test_row_0/C:col10/1731147474101/Put/seqid=0 2024-11-09T10:17:55,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742320_1496 (size=12301) 2024-11-09T10:17:55,821 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/68b68054d5044f5a88e061cf69fc42cf 2024-11-09T10:17:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/95d849836f8344fcbd5f3be5030586fe as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/95d849836f8344fcbd5f3be5030586fe 2024-11-09T10:17:55,828 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/95d849836f8344fcbd5f3be5030586fe, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T10:17:55,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/2ec57f520b0842d8a5b815ce741bf02c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2ec57f520b0842d8a5b815ce741bf02c 2024-11-09T10:17:55,831 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2ec57f520b0842d8a5b815ce741bf02c, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T10:17:55,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/68b68054d5044f5a88e061cf69fc42cf as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/68b68054d5044f5a88e061cf69fc42cf 2024-11-09T10:17:55,835 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/68b68054d5044f5a88e061cf69fc42cf, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T10:17:55,835 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9156a76309f41c5107dd18ab6eb49675 in 463ms, sequenceid=399, compaction requested=true 2024-11-09T10:17:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-09T10:17:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-09T10:17:55,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-09T10:17:55,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-09T10:17:55,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5380 sec 2024-11-09T10:17:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:55,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T10:17:55,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:55,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:55,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:55,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:55,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:55,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:55,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/686e7d624f2d41858d909734bda55e28 is 50, key is test_row_0/A:col10/1731147475512/Put/seqid=0 2024-11-09T10:17:55,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742321_1497 (size=14741) 2024-11-09T10:17:55,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/686e7d624f2d41858d909734bda55e28 2024-11-09T10:17:55,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/fa9094d94b7b4870bee7455efcfba4a9 is 50, key is test_row_0/B:col10/1731147475512/Put/seqid=0 2024-11-09T10:17:55,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742322_1498 (size=12301) 2024-11-09T10:17:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147535935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:56,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147536049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:56,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58382 deadline: 1731147536259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:56,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/fa9094d94b7b4870bee7455efcfba4a9 2024-11-09T10:17:56,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/bac7d30294904a73a505a48891a9b336 is 50, key is test_row_0/C:col10/1731147475512/Put/seqid=0 2024-11-09T10:17:56,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742323_1499 (size=12301) 2024-11-09T10:17:56,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/bac7d30294904a73a505a48891a9b336 2024-11-09T10:17:56,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/686e7d624f2d41858d909734bda55e28 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/686e7d624f2d41858d909734bda55e28 2024-11-09T10:17:56,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/686e7d624f2d41858d909734bda55e28, entries=200, sequenceid=419, filesize=14.4 K 2024-11-09T10:17:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/fa9094d94b7b4870bee7455efcfba4a9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/fa9094d94b7b4870bee7455efcfba4a9 2024-11-09T10:17:56,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/fa9094d94b7b4870bee7455efcfba4a9, entries=150, sequenceid=419, filesize=12.0 K 2024-11-09T10:17:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58414 deadline: 1731147536300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:56,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58376 deadline: 1731147536300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:56,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:56,311 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_1' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58424 deadline: 1731147536300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:56,311 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:56,312 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_0' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:56,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/bac7d30294904a73a505a48891a9b336 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/bac7d30294904a73a505a48891a9b336 2024-11-09T10:17:56,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:17:56,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/bac7d30294904a73a505a48891a9b336, entries=150, sequenceid=419, filesize=12.0 K 2024-11-09T10:17:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58398 deadline: 1731147536309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:56,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9156a76309f41c5107dd18ab6eb49675 in 461ms, sequenceid=419, compaction requested=true 2024-11-09T10:17:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:56,317 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:17:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:56,317 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in 9156a76309f41c5107dd18ab6eb49675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:17:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:17:56,317 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:56,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 , the old value is region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:56,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:17:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9156a76309f41c5107dd18ab6eb49675:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:17:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:56,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=2 because the exception is null or not the one we care about 2024-11-09T10:17:56,320 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:56,320 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/B is initiating minor compaction (all files) 2024-11-09T10:17:56,320 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/B in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:56,320 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/216723f40aea42b28629d6c5e7c1eb51, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2cc9942230584ea19f27968fa8742b57, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2ec57f520b0842d8a5b815ce741bf02c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/fa9094d94b7b4870bee7455efcfba4a9] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=48.8 K 2024-11-09T10:17:56,321 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:56,321 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/A is initiating minor compaction (all files) 2024-11-09T10:17:56,321 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/A in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:56,321 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/59ae61992c534f70bad58add2b282008, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/18a8471434f14ab9bf71638b8b90b54e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/95d849836f8344fcbd5f3be5030586fe, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/686e7d624f2d41858d909734bda55e28] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=53.6 K 2024-11-09T10:17:56,321 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 216723f40aea42b28629d6c5e7c1eb51, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731147472263 2024-11-09T10:17:56,322 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ae61992c534f70bad58add2b282008, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731147472263 2024-11-09T10:17:56,322 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc9942230584ea19f27968fa8742b57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731147472938 2024-11-09T10:17:56,322 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a8471434f14ab9bf71638b8b90b54e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731147472938 2024-11-09T10:17:56,322 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec57f520b0842d8a5b815ce741bf02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731147474101 2024-11-09T10:17:56,322 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d849836f8344fcbd5f3be5030586fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731147474101 2024-11-09T10:17:56,322 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9094d94b7b4870bee7455efcfba4a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731147475467 2024-11-09T10:17:56,323 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686e7d624f2d41858d909734bda55e28, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731147475467 2024-11-09T10:17:56,340 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:56,340 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:56,341 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/5609590f475c4fb297961867db3e40a9 is 50, key is test_row_0/B:col10/1731147475512/Put/seqid=0 2024-11-09T10:17:56,341 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/7e770c8a5f8242d68ae3364ea1472f06 is 50, key is test_row_0/A:col10/1731147475512/Put/seqid=0 2024-11-09T10:17:56,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742324_1500 (size=13221) 2024-11-09T10:17:56,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742325_1501 (size=13221) 2024-11-09T10:17:56,360 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/5609590f475c4fb297961867db3e40a9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5609590f475c4fb297961867db3e40a9 2024-11-09T10:17:56,360 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/7e770c8a5f8242d68ae3364ea1472f06 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/7e770c8a5f8242d68ae3364ea1472f06 2024-11-09T10:17:56,368 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/B of 9156a76309f41c5107dd18ab6eb49675 into 5609590f475c4fb297961867db3e40a9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:56,368 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:56,368 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/B, priority=12, startTime=1731147476317; duration=0sec 2024-11-09T10:17:56,368 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:17:56,368 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:B 2024-11-09T10:17:56,368 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:17:56,369 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/A of 9156a76309f41c5107dd18ab6eb49675 into 7e770c8a5f8242d68ae3364ea1472f06(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:56,369 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:56,369 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/A, priority=12, startTime=1731147476317; duration=0sec 2024-11-09T10:17:56,369 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:56,369 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:A 2024-11-09T10:17:56,371 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:17:56,371 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): 9156a76309f41c5107dd18ab6eb49675/C is initiating minor compaction (all files) 2024-11-09T10:17:56,371 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9156a76309f41c5107dd18ab6eb49675/C in TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:56,371 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/70888836e2a942f9bdc4fa754e9a2edf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/21d47d71ceef422783989b839e3366d5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/68b68054d5044f5a88e061cf69fc42cf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/bac7d30294904a73a505a48891a9b336] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp, totalSize=48.8 K 2024-11-09T10:17:56,371 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 70888836e2a942f9bdc4fa754e9a2edf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731147472263 2024-11-09T10:17:56,372 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d47d71ceef422783989b839e3366d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731147472938 2024-11-09T10:17:56,372 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 68b68054d5044f5a88e061cf69fc42cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731147474101 2024-11-09T10:17:56,372 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bac7d30294904a73a505a48891a9b336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731147475467 2024-11-09T10:17:56,379 INFO [Thread-1906 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1906. 2024-11-09T10:17:56,379 DEBUG [Thread-1906 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:56,379 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,379 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:56,383 INFO [Thread-1908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1908. 2024-11-09T10:17:56,383 DEBUG [Thread-1908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:56,383 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,383 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:56,386 INFO [Thread-1910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1910. 2024-11-09T10:17:56,386 INFO [Thread-1902 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1902. 2024-11-09T10:17:56,386 DEBUG [Thread-1910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:56,386 DEBUG [Thread-1902 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:56,386 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,386 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,386 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:56,386 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:56,389 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9156a76309f41c5107dd18ab6eb49675#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:17:56,389 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/b6292e415e434c118ed6637a2896680c is 50, key is test_row_0/C:col10/1731147475512/Put/seqid=0 2024-11-09T10:17:56,390 INFO [Thread-1904 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1904. 2024-11-09T10:17:56,390 DEBUG [Thread-1904 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:56,390 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,391 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:56,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742326_1502 (size=13221) 2024-11-09T10:17:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T10:17:56,449 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:17:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:56,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:17:56,580 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-11-09T10:17:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:56,580 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:56,580 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:56,580 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:56,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:56,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/45a5aa470ec744eb85bfee17bf40ed39 is 50, key is test_row_0/A:col10/1731147475896/Put/seqid=0 2024-11-09T10:17:56,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742327_1503 (size=12301) 2024-11-09T10:17:56,804 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/b6292e415e434c118ed6637a2896680c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/b6292e415e434c118ed6637a2896680c 2024-11-09T10:17:56,809 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9156a76309f41c5107dd18ab6eb49675/C of 9156a76309f41c5107dd18ab6eb49675 into b6292e415e434c118ed6637a2896680c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:17:56,809 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:56,809 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., storeName=9156a76309f41c5107dd18ab6eb49675/C, priority=12, startTime=1731147476318; duration=0sec 2024-11-09T10:17:56,809 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:17:56,809 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9156a76309f41c5107dd18ab6eb49675:C 2024-11-09T10:17:56,957 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T10:17:56,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/45a5aa470ec744eb85bfee17bf40ed39 2024-11-09T10:17:56,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/ffebe67aac1f4bb88e7ff79a381e7258 is 50, key is test_row_0/B:col10/1731147475896/Put/seqid=0 2024-11-09T10:17:56,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742328_1504 (size=12301) 2024-11-09T10:17:57,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/ffebe67aac1f4bb88e7ff79a381e7258 2024-11-09T10:17:57,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/429dcb31e78f4045809c570b6b9ee27f is 50, key is test_row_0/C:col10/1731147475896/Put/seqid=0 2024-11-09T10:17:57,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742329_1505 (size=12301) 2024-11-09T10:17:57,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/429dcb31e78f4045809c570b6b9ee27f 2024-11-09T10:17:57,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/45a5aa470ec744eb85bfee17bf40ed39 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/45a5aa470ec744eb85bfee17bf40ed39 2024-11-09T10:17:57,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/45a5aa470ec744eb85bfee17bf40ed39, entries=150, sequenceid=434, filesize=12.0 K 2024-11-09T10:17:57,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/ffebe67aac1f4bb88e7ff79a381e7258 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/ffebe67aac1f4bb88e7ff79a381e7258 2024-11-09T10:17:57,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/ffebe67aac1f4bb88e7ff79a381e7258, entries=150, sequenceid=434, filesize=12.0 K 2024-11-09T10:17:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/429dcb31e78f4045809c570b6b9ee27f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/429dcb31e78f4045809c570b6b9ee27f 2024-11-09T10:17:57,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/429dcb31e78f4045809c570b6b9ee27f, entries=150, sequenceid=434, filesize=12.0 K 2024-11-09T10:17:57,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 9156a76309f41c5107dd18ab6eb49675 in 1260ms, sequenceid=434, compaction requested=false 2024-11-09T10:17:57,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9156a76309f41c5107dd18ab6eb49675: 2024-11-09T10:17:58,330 INFO [Thread-1893 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1893. 2024-11-09T10:17:58,330 DEBUG [Thread-1893 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:58,330 DEBUG [Thread-1893 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,330 DEBUG [Thread-1893 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:58,339 INFO [Thread-1891 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1891. 2024-11-09T10:17:58,339 DEBUG [Thread-1891 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:58,340 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,340 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:58,340 INFO [Thread-1895 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1895. 2024-11-09T10:17:58,340 INFO [Thread-1897 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1897. 2024-11-09T10:17:58,340 DEBUG [Thread-1895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:58,340 DEBUG [Thread-1897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:17:58,340 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,340 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,340 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,340 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:58,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1152 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3451 rows 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1158 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3474 rows 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1152 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3456 rows 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3463 rows 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1151 2024-11-09T10:17:58,340 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3447 rows 2024-11-09T10:17:58,340 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:17:58,340 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:17:58,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:17:58,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:17:58,341 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T10:17:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T10:17:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T10:17:58,344 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147478344"}]},"ts":"1731147478344"} 2024-11-09T10:17:58,345 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T10:17:58,345 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T10:17:58,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:17:58,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, UNASSIGN}] 2024-11-09T10:17:58,347 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, UNASSIGN 2024-11-09T10:17:58,347 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=9156a76309f41c5107dd18ab6eb49675, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:17:58,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, UNASSIGN because future has completed 2024-11-09T10:17:58,349 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:17:58,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:17:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T10:17:58,500 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 9156a76309f41c5107dd18ab6eb49675, disabling compactions & flushes 2024-11-09T10:17:58,501 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. after waiting 0 ms 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:58,501 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 9156a76309f41c5107dd18ab6eb49675 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=A 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=B 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9156a76309f41c5107dd18ab6eb49675, store=C 2024-11-09T10:17:58,501 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:17:58,504 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/b265a87a5d58436c96dc35c131fb74f7 is 50, key is test_row_0/A:col10/1731147478339/Put/seqid=0 2024-11-09T10:17:58,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742330_1506 (size=12301) 2024-11-09T10:17:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T10:17:58,907 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/b265a87a5d58436c96dc35c131fb74f7 2024-11-09T10:17:58,912 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/9d2d9391b43540af9efb812ba5ad924d is 50, key is test_row_0/B:col10/1731147478339/Put/seqid=0 2024-11-09T10:17:58,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742331_1507 (size=12301) 2024-11-09T10:17:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T10:17:59,316 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/9d2d9391b43540af9efb812ba5ad924d 2024-11-09T10:17:59,321 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/8c4cc18392ab472d893f104d10ab47f1 is 50, key is test_row_0/C:col10/1731147478339/Put/seqid=0 2024-11-09T10:17:59,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742332_1508 (size=12301) 2024-11-09T10:17:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T10:17:59,724 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/8c4cc18392ab472d893f104d10ab47f1 2024-11-09T10:17:59,727 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/A/b265a87a5d58436c96dc35c131fb74f7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b265a87a5d58436c96dc35c131fb74f7 2024-11-09T10:17:59,730 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b265a87a5d58436c96dc35c131fb74f7, entries=150, sequenceid=442, filesize=12.0 K 2024-11-09T10:17:59,730 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/B/9d2d9391b43540af9efb812ba5ad924d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9d2d9391b43540af9efb812ba5ad924d 2024-11-09T10:17:59,733 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9d2d9391b43540af9efb812ba5ad924d, entries=150, sequenceid=442, filesize=12.0 K 2024-11-09T10:17:59,733 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/.tmp/C/8c4cc18392ab472d893f104d10ab47f1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8c4cc18392ab472d893f104d10ab47f1 2024-11-09T10:17:59,735 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8c4cc18392ab472d893f104d10ab47f1, entries=150, sequenceid=442, filesize=12.0 K 2024-11-09T10:17:59,736 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9156a76309f41c5107dd18ab6eb49675 in 1235ms, sequenceid=442, compaction requested=true 2024-11-09T10:17:59,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a08a6b1b50be4c859c36b4cb7070a999, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/fab6bb4f774f47a38c0783433ecd73f8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6ed50ac02efd449aa0a3ad63d87a6268, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d34886d52442c58d9275b78c011e9b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/08db3538eb1b471f88fdfd0762ffbc7f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/4e4616ed6bdd45cf93a5621f270a6531, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/016c617068234211a10e99ac953ef100, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d2c5eaa4074adb99288ae0ad24cdd6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/d7f9591ea0aa42b4bbe5e570d5fdc532, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c917b8a5c7804f14a926457afa4d2f9e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/5201bcf9d3cc48b9a66c1d4c958cecdf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/692b32514ff744eab195937767403024, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b72b1147a800481993819f42411a1b50, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a4c19e416e7f4b2ca30e4beb2e7e6a01, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/408fe0a4fe894a9d84f4afce56f223ad, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/53bc0533b9de45eaa2c80b941ee9fd59, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/bf0823755c634c3698e957e5ea2e2b2f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c6ab1189f7384a4895ee70d4f0598a3e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/86eb8936160d47de99400b5b08616555, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/e827e2b34e0f47afa4ada9f125862593, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/624f5839a16a4c1b888483536e37b450, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6bef92156b8d4f0ca94a8555df7408f1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/2548fc4a44d24e1aa8c28d5fa3f08ef4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/59ae61992c534f70bad58add2b282008, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/18a8471434f14ab9bf71638b8b90b54e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/95d849836f8344fcbd5f3be5030586fe, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/686e7d624f2d41858d909734bda55e28] to archive 2024-11-09T10:17:59,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:59,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a08a6b1b50be4c859c36b4cb7070a999 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a08a6b1b50be4c859c36b4cb7070a999 2024-11-09T10:17:59,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/fab6bb4f774f47a38c0783433ecd73f8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/fab6bb4f774f47a38c0783433ecd73f8 2024-11-09T10:17:59,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6ed50ac02efd449aa0a3ad63d87a6268 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6ed50ac02efd449aa0a3ad63d87a6268 2024-11-09T10:17:59,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d34886d52442c58d9275b78c011e9b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d34886d52442c58d9275b78c011e9b 2024-11-09T10:17:59,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/08db3538eb1b471f88fdfd0762ffbc7f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/08db3538eb1b471f88fdfd0762ffbc7f 2024-11-09T10:17:59,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/4e4616ed6bdd45cf93a5621f270a6531 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/4e4616ed6bdd45cf93a5621f270a6531 2024-11-09T10:17:59,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/016c617068234211a10e99ac953ef100 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/016c617068234211a10e99ac953ef100 2024-11-09T10:17:59,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d2c5eaa4074adb99288ae0ad24cdd6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a3d2c5eaa4074adb99288ae0ad24cdd6 2024-11-09T10:17:59,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/d7f9591ea0aa42b4bbe5e570d5fdc532 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/d7f9591ea0aa42b4bbe5e570d5fdc532 2024-11-09T10:17:59,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c917b8a5c7804f14a926457afa4d2f9e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c917b8a5c7804f14a926457afa4d2f9e 2024-11-09T10:17:59,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/5201bcf9d3cc48b9a66c1d4c958cecdf to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/5201bcf9d3cc48b9a66c1d4c958cecdf 2024-11-09T10:17:59,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/692b32514ff744eab195937767403024 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/692b32514ff744eab195937767403024 2024-11-09T10:17:59,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b72b1147a800481993819f42411a1b50 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b72b1147a800481993819f42411a1b50 2024-11-09T10:17:59,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a4c19e416e7f4b2ca30e4beb2e7e6a01 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/a4c19e416e7f4b2ca30e4beb2e7e6a01 2024-11-09T10:17:59,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/408fe0a4fe894a9d84f4afce56f223ad to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/408fe0a4fe894a9d84f4afce56f223ad 2024-11-09T10:17:59,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/53bc0533b9de45eaa2c80b941ee9fd59 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/53bc0533b9de45eaa2c80b941ee9fd59 2024-11-09T10:17:59,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/bf0823755c634c3698e957e5ea2e2b2f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/bf0823755c634c3698e957e5ea2e2b2f 2024-11-09T10:17:59,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c6ab1189f7384a4895ee70d4f0598a3e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/c6ab1189f7384a4895ee70d4f0598a3e 2024-11-09T10:17:59,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/86eb8936160d47de99400b5b08616555 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/86eb8936160d47de99400b5b08616555 2024-11-09T10:17:59,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/e827e2b34e0f47afa4ada9f125862593 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/e827e2b34e0f47afa4ada9f125862593 2024-11-09T10:17:59,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/624f5839a16a4c1b888483536e37b450 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/624f5839a16a4c1b888483536e37b450 2024-11-09T10:17:59,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6bef92156b8d4f0ca94a8555df7408f1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/6bef92156b8d4f0ca94a8555df7408f1 2024-11-09T10:17:59,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/2548fc4a44d24e1aa8c28d5fa3f08ef4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/2548fc4a44d24e1aa8c28d5fa3f08ef4 2024-11-09T10:17:59,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/59ae61992c534f70bad58add2b282008 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/59ae61992c534f70bad58add2b282008 2024-11-09T10:17:59,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/18a8471434f14ab9bf71638b8b90b54e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/18a8471434f14ab9bf71638b8b90b54e 2024-11-09T10:17:59,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/95d849836f8344fcbd5f3be5030586fe to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/95d849836f8344fcbd5f3be5030586fe 2024-11-09T10:17:59,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/686e7d624f2d41858d909734bda55e28 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/686e7d624f2d41858d909734bda55e28 2024-11-09T10:17:59,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5af913e8b99c4337a39748512dd75227, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/6db2f888197c4ef785f6c81dc56e9945, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/03c25664ddb747568f3775698bba0049, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/84881c4a7967413db058b4c552d4ffea, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/52aea315348c4b6c90856985a9c59a38, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/13f5c84a1ea243908a5894ecdd313e36, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7429090771a84248a8abc0a677cd1e6f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/bae2d23fe07844de8ae82ee2f8042c78, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0438a78617674a23b33bf85fc4fe1f8b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9a3040d3142b4b748caee0073c33a2e0, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/4809fabdab4843a095bb3bc1fe37001d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b56d5d0fab1f453c948cce7074a9fa03, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/94ea3cc2216e4e4f9d4511d86df38b5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/24437dd321444763ae395bc9819fb469, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/e691d39843e84d8c9edfc72bdfb8c9bc, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2475ac65fc314a2a8b1829d6aceaec6b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/3a89e4ea3d594a22ae04d5969accf87b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c3973840a96343469e303779b4306563, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/1fff6ec33cfb4602ae2af10b66ea5cb1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c22b64ef60dc4fa2a8ef6d9178282270, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0d0df719780a4bcfa7ae7cb11a751496, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b76fd7e43acd4129a9c921bc85f7165b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/216723f40aea42b28629d6c5e7c1eb51, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7080895927ab43eeae978877866768e8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2cc9942230584ea19f27968fa8742b57, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2ec57f520b0842d8a5b815ce741bf02c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/fa9094d94b7b4870bee7455efcfba4a9] to archive 2024-11-09T10:17:59,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:59,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5af913e8b99c4337a39748512dd75227 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5af913e8b99c4337a39748512dd75227 2024-11-09T10:17:59,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/6db2f888197c4ef785f6c81dc56e9945 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/6db2f888197c4ef785f6c81dc56e9945 2024-11-09T10:17:59,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/03c25664ddb747568f3775698bba0049 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/03c25664ddb747568f3775698bba0049 2024-11-09T10:17:59,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/84881c4a7967413db058b4c552d4ffea to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/84881c4a7967413db058b4c552d4ffea 2024-11-09T10:17:59,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/52aea315348c4b6c90856985a9c59a38 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/52aea315348c4b6c90856985a9c59a38 2024-11-09T10:17:59,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/13f5c84a1ea243908a5894ecdd313e36 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/13f5c84a1ea243908a5894ecdd313e36 2024-11-09T10:17:59,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7429090771a84248a8abc0a677cd1e6f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7429090771a84248a8abc0a677cd1e6f 2024-11-09T10:17:59,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/bae2d23fe07844de8ae82ee2f8042c78 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/bae2d23fe07844de8ae82ee2f8042c78 2024-11-09T10:17:59,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0438a78617674a23b33bf85fc4fe1f8b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0438a78617674a23b33bf85fc4fe1f8b 2024-11-09T10:17:59,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9a3040d3142b4b748caee0073c33a2e0 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9a3040d3142b4b748caee0073c33a2e0 2024-11-09T10:17:59,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/4809fabdab4843a095bb3bc1fe37001d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/4809fabdab4843a095bb3bc1fe37001d 2024-11-09T10:17:59,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b56d5d0fab1f453c948cce7074a9fa03 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b56d5d0fab1f453c948cce7074a9fa03 2024-11-09T10:17:59,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/94ea3cc2216e4e4f9d4511d86df38b5c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/94ea3cc2216e4e4f9d4511d86df38b5c 2024-11-09T10:17:59,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/24437dd321444763ae395bc9819fb469 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/24437dd321444763ae395bc9819fb469 2024-11-09T10:17:59,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/e691d39843e84d8c9edfc72bdfb8c9bc to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/e691d39843e84d8c9edfc72bdfb8c9bc 2024-11-09T10:17:59,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2475ac65fc314a2a8b1829d6aceaec6b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2475ac65fc314a2a8b1829d6aceaec6b 2024-11-09T10:17:59,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/3a89e4ea3d594a22ae04d5969accf87b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/3a89e4ea3d594a22ae04d5969accf87b 2024-11-09T10:17:59,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c3973840a96343469e303779b4306563 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c3973840a96343469e303779b4306563 2024-11-09T10:17:59,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/1fff6ec33cfb4602ae2af10b66ea5cb1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/1fff6ec33cfb4602ae2af10b66ea5cb1 2024-11-09T10:17:59,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c22b64ef60dc4fa2a8ef6d9178282270 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/c22b64ef60dc4fa2a8ef6d9178282270 2024-11-09T10:17:59,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0d0df719780a4bcfa7ae7cb11a751496 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/0d0df719780a4bcfa7ae7cb11a751496 2024-11-09T10:17:59,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b76fd7e43acd4129a9c921bc85f7165b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/b76fd7e43acd4129a9c921bc85f7165b 2024-11-09T10:17:59,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/216723f40aea42b28629d6c5e7c1eb51 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/216723f40aea42b28629d6c5e7c1eb51 2024-11-09T10:17:59,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7080895927ab43eeae978877866768e8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/7080895927ab43eeae978877866768e8 2024-11-09T10:17:59,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2cc9942230584ea19f27968fa8742b57 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2cc9942230584ea19f27968fa8742b57 2024-11-09T10:17:59,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2ec57f520b0842d8a5b815ce741bf02c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/2ec57f520b0842d8a5b815ce741bf02c 2024-11-09T10:17:59,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/fa9094d94b7b4870bee7455efcfba4a9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/fa9094d94b7b4870bee7455efcfba4a9 2024-11-09T10:17:59,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/5567230c2ed249ffae8b1ab24e0c5f5a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/0783b322074e4cfeba4cce445dae4260, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/2aed0688a3d8418abff9c7b47b9bccee, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e29fc00bcae647bba229f8526700b37c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/18dfb3aad3dc4c0f9adaa2c72de70157, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed7392980dfc4f9c90515b4de1a03f4d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/1e86d6ec8a2e410bb59466d6a761ab46, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed06b277c216454fb3058467bf15444f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/261a561c0ae348a4a7b708426b465069, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/19738e0a42c8426facdf9bebeb684667, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/df050acdd0c74b5493369353f112a449, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e97f5a13d481482da0c3600d64490e5d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/6ca2eed4425d46479ba9cf2d87d226e4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/a02e1b0eee0b4e458b49713c9f3fd9f6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/c832de8ad1ff4242b88f066119cd38d4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8652acc9488c4d0294edb8ab0d5d0e5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/db27de0e23304452b9eaebe0d43bbcc8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/acd2375e73c6444199eee66ab106d6e5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e8891c51d2744b58ad1669829ae1e9ac, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/46f87ca3fc434294880d889187940795, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/4bc5504a7c5641b495ef13819a11dcd9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/dfd1bd5f9ad8402dae8f60c5f3627285, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/70888836e2a942f9bdc4fa754e9a2edf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/672461a8008c49918966c880b2c9c583, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/21d47d71ceef422783989b839e3366d5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/68b68054d5044f5a88e061cf69fc42cf, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/bac7d30294904a73a505a48891a9b336] to archive 2024-11-09T10:17:59,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:17:59,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/5567230c2ed249ffae8b1ab24e0c5f5a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/5567230c2ed249ffae8b1ab24e0c5f5a 2024-11-09T10:17:59,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/0783b322074e4cfeba4cce445dae4260 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/0783b322074e4cfeba4cce445dae4260 2024-11-09T10:17:59,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/2aed0688a3d8418abff9c7b47b9bccee to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/2aed0688a3d8418abff9c7b47b9bccee 2024-11-09T10:17:59,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e29fc00bcae647bba229f8526700b37c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e29fc00bcae647bba229f8526700b37c 2024-11-09T10:17:59,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/18dfb3aad3dc4c0f9adaa2c72de70157 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/18dfb3aad3dc4c0f9adaa2c72de70157 2024-11-09T10:17:59,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed7392980dfc4f9c90515b4de1a03f4d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed7392980dfc4f9c90515b4de1a03f4d 2024-11-09T10:17:59,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/1e86d6ec8a2e410bb59466d6a761ab46 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/1e86d6ec8a2e410bb59466d6a761ab46 2024-11-09T10:17:59,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed06b277c216454fb3058467bf15444f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/ed06b277c216454fb3058467bf15444f 2024-11-09T10:17:59,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/261a561c0ae348a4a7b708426b465069 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/261a561c0ae348a4a7b708426b465069 2024-11-09T10:17:59,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/19738e0a42c8426facdf9bebeb684667 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/19738e0a42c8426facdf9bebeb684667 2024-11-09T10:17:59,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/df050acdd0c74b5493369353f112a449 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/df050acdd0c74b5493369353f112a449 2024-11-09T10:17:59,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e97f5a13d481482da0c3600d64490e5d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e97f5a13d481482da0c3600d64490e5d 2024-11-09T10:17:59,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/6ca2eed4425d46479ba9cf2d87d226e4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/6ca2eed4425d46479ba9cf2d87d226e4 2024-11-09T10:17:59,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/a02e1b0eee0b4e458b49713c9f3fd9f6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/a02e1b0eee0b4e458b49713c9f3fd9f6 2024-11-09T10:17:59,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/c832de8ad1ff4242b88f066119cd38d4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/c832de8ad1ff4242b88f066119cd38d4 2024-11-09T10:17:59,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8652acc9488c4d0294edb8ab0d5d0e5c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8652acc9488c4d0294edb8ab0d5d0e5c 2024-11-09T10:17:59,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/db27de0e23304452b9eaebe0d43bbcc8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/db27de0e23304452b9eaebe0d43bbcc8 2024-11-09T10:17:59,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/acd2375e73c6444199eee66ab106d6e5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/acd2375e73c6444199eee66ab106d6e5 2024-11-09T10:17:59,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e8891c51d2744b58ad1669829ae1e9ac to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/e8891c51d2744b58ad1669829ae1e9ac 2024-11-09T10:17:59,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/46f87ca3fc434294880d889187940795 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/46f87ca3fc434294880d889187940795 2024-11-09T10:17:59,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/4bc5504a7c5641b495ef13819a11dcd9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/4bc5504a7c5641b495ef13819a11dcd9 2024-11-09T10:17:59,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/dfd1bd5f9ad8402dae8f60c5f3627285 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/dfd1bd5f9ad8402dae8f60c5f3627285 2024-11-09T10:17:59,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/70888836e2a942f9bdc4fa754e9a2edf to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/70888836e2a942f9bdc4fa754e9a2edf 2024-11-09T10:17:59,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/672461a8008c49918966c880b2c9c583 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/672461a8008c49918966c880b2c9c583 2024-11-09T10:17:59,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/21d47d71ceef422783989b839e3366d5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/21d47d71ceef422783989b839e3366d5 2024-11-09T10:17:59,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/68b68054d5044f5a88e061cf69fc42cf to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/68b68054d5044f5a88e061cf69fc42cf 2024-11-09T10:17:59,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/bac7d30294904a73a505a48891a9b336 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/bac7d30294904a73a505a48891a9b336 2024-11-09T10:17:59,795 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=1 2024-11-09T10:17:59,795 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. 2024-11-09T10:17:59,795 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 9156a76309f41c5107dd18ab6eb49675: Waiting for close lock at 1731147478501Running coprocessor pre-close hooks at 1731147478501Disabling compacts and flushes for region at 1731147478501Disabling writes for close at 1731147478501Obtaining lock to block concurrent updates at 1731147478501Preparing flush snapshotting stores in 9156a76309f41c5107dd18ab6eb49675 at 1731147478501Finished memstore snapshotting TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731147478501Flushing stores of TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675. at 1731147478502 (+1 ms)Flushing 9156a76309f41c5107dd18ab6eb49675/A: creating writer at 1731147478502Flushing 9156a76309f41c5107dd18ab6eb49675/A: appending metadata at 1731147478503 (+1 ms)Flushing 9156a76309f41c5107dd18ab6eb49675/A: closing flushed file at 1731147478503Flushing 9156a76309f41c5107dd18ab6eb49675/B: creating writer at 1731147478910 (+407 ms)Flushing 9156a76309f41c5107dd18ab6eb49675/B: appending metadata at 1731147478912 (+2 ms)Flushing 9156a76309f41c5107dd18ab6eb49675/B: closing flushed file at 1731147478912Flushing 9156a76309f41c5107dd18ab6eb49675/C: creating writer at 1731147479319 (+407 ms)Flushing 9156a76309f41c5107dd18ab6eb49675/C: appending metadata at 1731147479321 (+2 ms)Flushing 9156a76309f41c5107dd18ab6eb49675/C: closing flushed file at 1731147479321Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57080339: reopening flushed file at 1731147479727 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1480b8: reopening flushed file at 1731147479730 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20672462: reopening flushed file at 1731147479733 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9156a76309f41c5107dd18ab6eb49675 in 1235ms, sequenceid=442, compaction requested=true at 1731147479736 (+3 ms)Writing region close event to WAL at 1731147479793 (+57 ms)Running coprocessor post-close hooks at 1731147479795 (+2 ms)Closed at 1731147479795 2024-11-09T10:17:59,797 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:17:59,797 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=9156a76309f41c5107dd18ab6eb49675, regionState=CLOSED 2024-11-09T10:17:59,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:17:59,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-09T10:17:59,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 9156a76309f41c5107dd18ab6eb49675, server=e067b2c2c7b3,38427,1731147328756 in 1.4500 sec 2024-11-09T10:17:59,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-09T10:17:59,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9156a76309f41c5107dd18ab6eb49675, UNASSIGN in 1.4540 sec 2024-11-09T10:17:59,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-09T10:17:59,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4570 sec 2024-11-09T10:17:59,804 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147479804"}]},"ts":"1731147479804"} 2024-11-09T10:17:59,805 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T10:17:59,805 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T10:17:59,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4640 sec 2024-11-09T10:18:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T10:18:00,489 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:00,490 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T10:18:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:00,491 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T10:18:00,492 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:00,494 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:18:00,495 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/recovered.edits] 2024-11-09T10:18:00,497 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/45a5aa470ec744eb85bfee17bf40ed39 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/45a5aa470ec744eb85bfee17bf40ed39 2024-11-09T10:18:00,498 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/7e770c8a5f8242d68ae3364ea1472f06 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/7e770c8a5f8242d68ae3364ea1472f06 2024-11-09T10:18:00,499 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b265a87a5d58436c96dc35c131fb74f7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/A/b265a87a5d58436c96dc35c131fb74f7 2024-11-09T10:18:00,500 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5609590f475c4fb297961867db3e40a9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/5609590f475c4fb297961867db3e40a9 2024-11-09T10:18:00,501 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9d2d9391b43540af9efb812ba5ad924d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/9d2d9391b43540af9efb812ba5ad924d 2024-11-09T10:18:00,502 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/ffebe67aac1f4bb88e7ff79a381e7258 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/B/ffebe67aac1f4bb88e7ff79a381e7258 2024-11-09T10:18:00,503 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/429dcb31e78f4045809c570b6b9ee27f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/429dcb31e78f4045809c570b6b9ee27f 2024-11-09T10:18:00,504 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8c4cc18392ab472d893f104d10ab47f1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/8c4cc18392ab472d893f104d10ab47f1 2024-11-09T10:18:00,505 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/b6292e415e434c118ed6637a2896680c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/C/b6292e415e434c118ed6637a2896680c 2024-11-09T10:18:00,507 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/recovered.edits/445.seqid to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675/recovered.edits/445.seqid 2024-11-09T10:18:00,507 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/9156a76309f41c5107dd18ab6eb49675 2024-11-09T10:18:00,507 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T10:18:00,509 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:00,512 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T10:18:00,514 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T10:18:00,515 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:00,515 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T10:18:00,515 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731147480515"}]},"ts":"9223372036854775807"} 2024-11-09T10:18:00,517 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T10:18:00,517 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9156a76309f41c5107dd18ab6eb49675, NAME => 'TestAcidGuarantees,,1731147454166.9156a76309f41c5107dd18ab6eb49675.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T10:18:00,517 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T10:18:00,517 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731147480517"}]},"ts":"9223372036854775807"} 2024-11-09T10:18:00,518 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T10:18:00,519 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:00,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-09T10:18:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T10:18:00,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T10:18:00,599 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:00,607 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 553), ProcessCount=11 (was 11), AvailableMemoryMB=1347 (was 1848) 2024-11-09T10:18:00,614 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=1347 2024-11-09T10:18:00,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:18:00,616 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:18:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:00,617 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T10:18:00,617 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:00,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-09T10:18:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T10:18:00,618 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T10:18:00,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742333_1509 (size=1061) 2024-11-09T10:18:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T10:18:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T10:18:01,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51 2024-11-09T10:18:01,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742334_1510 (size=53) 2024-11-09T10:18:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T10:18:01,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:18:01,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing be37ddd6456db463b3a7fa388151b50e, disabling compactions & flushes 2024-11-09T10:18:01,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. after waiting 0 ms 2024-11-09T10:18:01,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for be37ddd6456db463b3a7fa388151b50e: Waiting for close lock at 1731147481429Disabling compacts and flushes for region at 1731147481429Disabling writes for close at 1731147481429Writing region close event to WAL at 1731147481429Closed at 1731147481429 2024-11-09T10:18:01,430 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T10:18:01,430 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731147481430"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731147481430"}]},"ts":"1731147481430"} 2024-11-09T10:18:01,431 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T10:18:01,432 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T10:18:01,432 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147481432"}]},"ts":"1731147481432"} 2024-11-09T10:18:01,433 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T10:18:01,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, ASSIGN}] 2024-11-09T10:18:01,434 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, ASSIGN 2024-11-09T10:18:01,434 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, ASSIGN; state=OFFLINE, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=false 2024-11-09T10:18:01,585 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:01,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, ASSIGN because future has completed 2024-11-09T10:18:01,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:18:01,740 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,740 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:18:01,741 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,741 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:18:01,741 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,741 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,742 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,743 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:18:01,743 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be37ddd6456db463b3a7fa388151b50e columnFamilyName A 2024-11-09T10:18:01,743 DEBUG [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:01,743 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(327): Store=be37ddd6456db463b3a7fa388151b50e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:18:01,743 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,744 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:18:01,744 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be37ddd6456db463b3a7fa388151b50e columnFamilyName B 2024-11-09T10:18:01,744 DEBUG [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:01,744 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(327): Store=be37ddd6456db463b3a7fa388151b50e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:18:01,744 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,745 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:18:01,745 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be37ddd6456db463b3a7fa388151b50e columnFamilyName C 2024-11-09T10:18:01,745 DEBUG [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:01,746 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(327): Store=be37ddd6456db463b3a7fa388151b50e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:18:01,746 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,746 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,746 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,746 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,747 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,747 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,748 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:18:01,749 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,750 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T10:18:01,750 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened be37ddd6456db463b3a7fa388151b50e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71364411, jitterRate=0.06341259181499481}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:18:01,750 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:01,751 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for be37ddd6456db463b3a7fa388151b50e: Running coprocessor pre-open hook at 1731147481741Writing region info on filesystem at 1731147481741Initializing all the Stores at 1731147481741Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147481741Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147481742 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147481742Cleaning up temporary data from old regions at 1731147481747 (+5 ms)Running coprocessor post-open hooks at 1731147481750 (+3 ms)Region opened successfully at 1731147481751 (+1 ms) 2024-11-09T10:18:01,751 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., pid=140, masterSystemTime=1731147481738 2024-11-09T10:18:01,752 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,753 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:01,753 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=OPEN, openSeqNum=2, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:01,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:18:01,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-09T10:18:01,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 in 169 msec 2024-11-09T10:18:01,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-09T10:18:01,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, ASSIGN in 323 msec 2024-11-09T10:18:01,758 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T10:18:01,758 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147481758"}]},"ts":"1731147481758"} 2024-11-09T10:18:01,759 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T10:18:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T10:18:01,760 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T10:18:01,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1440 sec 2024-11-09T10:18:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T10:18:02,769 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:02,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae4ba4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:02,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:02,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:02,771 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5120e5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:02,773 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae1b6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:02,774 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:02,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:02,776 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T10:18:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e067b2c2c7b3,42133,1731147327850 2024-11-09T10:18:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3dc671e5 2024-11-09T10:18:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T10:18:02,778 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T10:18:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T10:18:02,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T10:18:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:02,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742335_1511 (size=1097) 2024-11-09T10:18:03,207 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T10:18:03,207 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T10:18:03,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:18:03,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, REOPEN/MOVE}] 2024-11-09T10:18:03,212 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, REOPEN/MOVE 2024-11-09T10:18:03,212 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, REOPEN/MOVE because future has completed 2024-11-09T10:18:03,214 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:18:03,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:18:03,365 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,365 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:18:03,366 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing be37ddd6456db463b3a7fa388151b50e, disabling compactions & flushes 2024-11-09T10:18:03,366 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,366 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,366 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. after waiting 0 ms 2024-11-09T10:18:03,366 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,369 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T10:18:03,369 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,369 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for be37ddd6456db463b3a7fa388151b50e: Waiting for close lock at 1731147483366Running coprocessor pre-close hooks at 1731147483366Disabling compacts and flushes for region at 1731147483366Disabling writes for close at 1731147483366Writing region close event to WAL at 1731147483366Running coprocessor post-close hooks at 1731147483369 (+3 ms)Closed at 1731147483369 2024-11-09T10:18:03,369 WARN [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: be37ddd6456db463b3a7fa388151b50e to self. 2024-11-09T10:18:03,371 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,371 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=CLOSED 2024-11-09T10:18:03,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:18:03,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-09T10:18:03,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 in 159 msec 2024-11-09T10:18:03,375 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, REOPEN/MOVE; state=CLOSED, location=e067b2c2c7b3,38427,1731147328756; forceNewPlan=false, retain=true 2024-11-09T10:18:03,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=OPENING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, REOPEN/MOVE because future has completed 2024-11-09T10:18:03,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:18:03,680 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,680 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T10:18:03,680 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,680 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T10:18:03,681 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,681 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,682 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,682 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:18:03,682 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be37ddd6456db463b3a7fa388151b50e columnFamilyName A 2024-11-09T10:18:03,683 DEBUG [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:03,683 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(327): Store=be37ddd6456db463b3a7fa388151b50e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:18:03,684 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,684 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:18:03,684 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be37ddd6456db463b3a7fa388151b50e columnFamilyName B 2024-11-09T10:18:03,684 DEBUG [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:03,685 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(327): Store=be37ddd6456db463b3a7fa388151b50e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:18:03,685 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,685 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T10:18:03,685 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be37ddd6456db463b3a7fa388151b50e columnFamilyName C 2024-11-09T10:18:03,685 DEBUG [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:03,685 INFO [StoreOpener-be37ddd6456db463b3a7fa388151b50e-1 {}] regionserver.HStore(327): Store=be37ddd6456db463b3a7fa388151b50e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T10:18:03,686 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,686 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,686 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,687 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,688 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,688 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,688 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T10:18:03,689 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,690 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened be37ddd6456db463b3a7fa388151b50e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62722487, jitterRate=-0.0653621107339859}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T10:18:03,690 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,690 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for be37ddd6456db463b3a7fa388151b50e: Running coprocessor pre-open hook at 1731147483681Writing region info on filesystem at 1731147483681Initializing all the Stores at 1731147483681Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147483681Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147483682 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731147483682Cleaning up temporary data from old regions at 1731147483688 (+6 ms)Running coprocessor post-open hooks at 1731147483690 (+2 ms)Region opened successfully at 1731147483690 2024-11-09T10:18:03,691 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., pid=145, masterSystemTime=1731147483678 2024-11-09T10:18:03,692 DEBUG [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,692 INFO [RS_OPEN_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,692 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=OPEN, openSeqNum=5, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:18:03,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-09T10:18:03,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 in 168 msec 2024-11-09T10:18:03,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-09T10:18:03,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, REOPEN/MOVE in 484 msec 2024-11-09T10:18:03,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-09T10:18:03,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-09T10:18:03,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-11-09T10:18:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-09T10:18:03,701 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:03,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615c30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0bf588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5407fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c6f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,705 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d35b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,706 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14381238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5ba127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8b6100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc41325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110f2d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169aed07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,711 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d5876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbb80ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,712 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d58dabd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,713 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eea7c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ae8872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,715 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769adcaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,716 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b995fdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf38cf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,718 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa5407c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c83b31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c3cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2620633b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,721 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771d90c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b067c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5280c1a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3ad42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebeeab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e067b2c2c7b3,42133,-1 for getting cluster id 2024-11-09T10:18:03,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T10:18:03,725 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4d84759-b9d8-4b81-ab08-fb22b416de6b' 2024-11-09T10:18:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T10:18:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4d84759-b9d8-4b81-ab08-fb22b416de6b" 2024-11-09T10:18:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df8bce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e067b2c2c7b3,42133,-1] 2024-11-09T10:18:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T10:18:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:03,726 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T10:18:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749725f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T10:18:03,727 DEBUG [Thread-2310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,727 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,727 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,727 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,727 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,727 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,727 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,728 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,729 DEBUG [Thread-2299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,729 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:03,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-09T10:18:03,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,730 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T10:18:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T10:18:03,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e067b2c2c7b3,38427,1731147328756, seqNum=-1] 2024-11-09T10:18:03,731 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T10:18:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T10:18:03,733 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:03,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5] 2024-11-09T10:18:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:03,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T10:18:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:03,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147543754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147543755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147543756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147543757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147543758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109da2817c451594814bba8e928e2cc46a6_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147483736/Put/seqid=0 2024-11-09T10:18:03,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742336_1512 (size=14594) 2024-11-09T10:18:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T10:18:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147543868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147543868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147543869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147543869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147543869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:03,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T10:18:03,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:03,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:04,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T10:18:04,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:04,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T10:18:04,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147544078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147544079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147544079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147544079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147544079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,186 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:04,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109da2817c451594814bba8e928e2cc46a6_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109da2817c451594814bba8e928e2cc46a6_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:04,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/82f6bc09d5d24d969aa4cefcef429a9a, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:04,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/82f6bc09d5d24d969aa4cefcef429a9a is 175, key is test_row_0/A:col10/1731147483736/Put/seqid=0 2024-11-09T10:18:04,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T10:18:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:04,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:04,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742337_1513 (size=39549) 2024-11-09T10:18:04,207 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/82f6bc09d5d24d969aa4cefcef429a9a 2024-11-09T10:18:04,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/96c5ae85acf24480b8e8dee445a3ad09 is 50, key is test_row_0/B:col10/1731147483736/Put/seqid=0 2024-11-09T10:18:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742338_1514 (size=12001) 2024-11-09T10:18:04,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/96c5ae85acf24480b8e8dee445a3ad09 2024-11-09T10:18:04,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/bdef70ec1511475197eeae351c080210 is 50, key is test_row_0/C:col10/1731147483736/Put/seqid=0 2024-11-09T10:18:04,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742339_1515 (size=12001) 2024-11-09T10:18:04,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/bdef70ec1511475197eeae351c080210 2024-11-09T10:18:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/82f6bc09d5d24d969aa4cefcef429a9a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/82f6bc09d5d24d969aa4cefcef429a9a 2024-11-09T10:18:04,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/82f6bc09d5d24d969aa4cefcef429a9a, entries=200, sequenceid=16, filesize=38.6 K 2024-11-09T10:18:04,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/96c5ae85acf24480b8e8dee445a3ad09 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/96c5ae85acf24480b8e8dee445a3ad09 2024-11-09T10:18:04,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/96c5ae85acf24480b8e8dee445a3ad09, entries=150, sequenceid=16, filesize=11.7 K 2024-11-09T10:18:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/bdef70ec1511475197eeae351c080210 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bdef70ec1511475197eeae351c080210 2024-11-09T10:18:04,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bdef70ec1511475197eeae351c080210, entries=150, sequenceid=16, filesize=11.7 K 2024-11-09T10:18:04,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for be37ddd6456db463b3a7fa388151b50e in 558ms, sequenceid=16, compaction requested=false 2024-11-09T10:18:04,298 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T10:18:04,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:04,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T10:18:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:04,350 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T10:18:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T10:18:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099c01420b04a64a1e89d445b42261fd66_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147483757/Put/seqid=0 2024-11-09T10:18:04,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742340_1516 (size=12154) 2024-11-09T10:18:04,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:04,382 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099c01420b04a64a1e89d445b42261fd66_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099c01420b04a64a1e89d445b42261fd66_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:04,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/b55a19b73da043298ce030e52f68f648, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:04,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/b55a19b73da043298ce030e52f68f648 is 175, key is test_row_0/A:col10/1731147483757/Put/seqid=0 2024-11-09T10:18:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:04,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:04,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742341_1517 (size=30955) 2024-11-09T10:18:04,392 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/b55a19b73da043298ce030e52f68f648 2024-11-09T10:18:04,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147544397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/9ab93d6e22254c899664426a4f169010 is 50, key is test_row_0/B:col10/1731147483757/Put/seqid=0 2024-11-09T10:18:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147544398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147544399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147544401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147544401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742342_1518 (size=12001) 2024-11-09T10:18:04,431 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/9ab93d6e22254c899664426a4f169010 2024-11-09T10:18:04,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/4e1674748b39499f8b46eb3b4580ed06 is 50, key is test_row_0/C:col10/1731147483757/Put/seqid=0 2024-11-09T10:18:04,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742343_1519 (size=12001) 2024-11-09T10:18:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147544509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147544509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147544509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147544509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147544509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147544719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147544719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147544719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147544719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147544719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:04,846 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/4e1674748b39499f8b46eb3b4580ed06 2024-11-09T10:18:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/b55a19b73da043298ce030e52f68f648 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/b55a19b73da043298ce030e52f68f648 2024-11-09T10:18:04,853 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/b55a19b73da043298ce030e52f68f648, entries=150, sequenceid=36, filesize=30.2 K 2024-11-09T10:18:04,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/9ab93d6e22254c899664426a4f169010 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9ab93d6e22254c899664426a4f169010 2024-11-09T10:18:04,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9ab93d6e22254c899664426a4f169010, entries=150, sequenceid=36, filesize=11.7 K 2024-11-09T10:18:04,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/4e1674748b39499f8b46eb3b4580ed06 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/4e1674748b39499f8b46eb3b4580ed06 2024-11-09T10:18:04,861 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/4e1674748b39499f8b46eb3b4580ed06, entries=150, sequenceid=36, filesize=11.7 K 2024-11-09T10:18:04,862 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for be37ddd6456db463b3a7fa388151b50e in 512ms, sequenceid=36, compaction requested=false 2024-11-09T10:18:04,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:04,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:04,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-09T10:18:04,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-09T10:18:04,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-09T10:18:04,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1310 sec 2024-11-09T10:18:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T10:18:04,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.1430 sec 2024-11-09T10:18:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:05,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:05,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f298b43646e645db858d38c0040b37b1_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147485030/Put/seqid=0 2024-11-09T10:18:05,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742344_1520 (size=17034) 2024-11-09T10:18:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147545043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147545043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147545043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147545044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147545046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147545158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147545159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147545159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147545159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147545159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,312 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T10:18:05,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147545368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147545369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147545369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147545369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147545369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:05,445 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f298b43646e645db858d38c0040b37b1_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f298b43646e645db858d38c0040b37b1_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:05,446 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/2b78d77a643643a79273c6e4618db847, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:05,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/2b78d77a643643a79273c6e4618db847 is 175, key is test_row_0/A:col10/1731147485030/Put/seqid=0 2024-11-09T10:18:05,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742345_1521 (size=48139) 2024-11-09T10:18:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147545679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147545679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147545679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147545679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147545679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:05,853 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/2b78d77a643643a79273c6e4618db847 2024-11-09T10:18:05,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/6cd2a2a6bfe849eeaa23d1f712030e67 is 50, key is test_row_0/B:col10/1731147485030/Put/seqid=0 2024-11-09T10:18:05,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742346_1522 (size=12001) 2024-11-09T10:18:05,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/6cd2a2a6bfe849eeaa23d1f712030e67 2024-11-09T10:18:05,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9 is 50, key is test_row_0/C:col10/1731147485030/Put/seqid=0 2024-11-09T10:18:05,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742347_1523 (size=12001) 2024-11-09T10:18:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T10:18:05,895 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:05,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-09T10:18:05,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T10:18:05,898 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:05,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T10:18:06,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T10:18:06,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:06,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:06,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147546189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147546189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147546189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147546189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147546189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:06,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T10:18:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:06,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T10:18:06,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9 2024-11-09T10:18:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/2b78d77a643643a79273c6e4618db847 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/2b78d77a643643a79273c6e4618db847 2024-11-09T10:18:06,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/2b78d77a643643a79273c6e4618db847, entries=250, sequenceid=51, filesize=47.0 K 2024-11-09T10:18:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/6cd2a2a6bfe849eeaa23d1f712030e67 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/6cd2a2a6bfe849eeaa23d1f712030e67 2024-11-09T10:18:06,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/6cd2a2a6bfe849eeaa23d1f712030e67, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T10:18:06,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9 2024-11-09T10:18:06,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T10:18:06,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for be37ddd6456db463b3a7fa388151b50e in 1259ms, sequenceid=51, compaction requested=true 2024-11-09T10:18:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:06,291 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:06,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:06,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:06,291 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:06,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:06,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:06,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:06,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:06,293 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:06,293 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:06,293 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,293 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/96c5ae85acf24480b8e8dee445a3ad09, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9ab93d6e22254c899664426a4f169010, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/6cd2a2a6bfe849eeaa23d1f712030e67] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=35.2 K 2024-11-09T10:18:06,293 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:06,293 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:06,293 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,293 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/82f6bc09d5d24d969aa4cefcef429a9a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/b55a19b73da043298ce030e52f68f648, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/2b78d77a643643a79273c6e4618db847] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=115.9 K 2024-11-09T10:18:06,293 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c5ae85acf24480b8e8dee445a3ad09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731147483736 2024-11-09T10:18:06,293 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,293 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/82f6bc09d5d24d969aa4cefcef429a9a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/b55a19b73da043298ce030e52f68f648, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/2b78d77a643643a79273c6e4618db847] 2024-11-09T10:18:06,294 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ab93d6e22254c899664426a4f169010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731147483751 2024-11-09T10:18:06,294 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f6bc09d5d24d969aa4cefcef429a9a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731147483736 2024-11-09T10:18:06,294 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting b55a19b73da043298ce030e52f68f648, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731147483751 2024-11-09T10:18:06,294 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd2a2a6bfe849eeaa23d1f712030e67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147484390 2024-11-09T10:18:06,294 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b78d77a643643a79273c6e4618db847, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147484390 2024-11-09T10:18:06,304 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:06,318 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:06,318 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/78fd1dd6f2314e468d8e8aef172ed50c is 50, key is test_row_0/B:col10/1731147485030/Put/seqid=0 2024-11-09T10:18:06,319 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110902fcfd5e1c564063a8a8cd97997203ec_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:06,322 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110902fcfd5e1c564063a8a8cd97997203ec_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:06,322 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110902fcfd5e1c564063a8a8cd97997203ec_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:06,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742349_1525 (size=4469) 2024-11-09T10:18:06,337 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#442 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:06,337 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/44412fb044a74ec5836b7e4ca1991f1d is 175, key is test_row_0/A:col10/1731147485030/Put/seqid=0 2024-11-09T10:18:06,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T10:18:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T10:18:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:06,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:06,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742348_1524 (size=12104) 2024-11-09T10:18:06,368 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/78fd1dd6f2314e468d8e8aef172ed50c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/78fd1dd6f2314e468d8e8aef172ed50c 2024-11-09T10:18:06,373 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into 78fd1dd6f2314e468d8e8aef172ed50c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:06,373 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:06,373 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=13, startTime=1731147486291; duration=0sec 2024-11-09T10:18:06,373 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:06,373 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:06,373 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:06,374 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:06,374 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:06,374 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,374 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bdef70ec1511475197eeae351c080210, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/4e1674748b39499f8b46eb3b4580ed06, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=35.2 K 2024-11-09T10:18:06,374 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting bdef70ec1511475197eeae351c080210, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731147483736 2024-11-09T10:18:06,374 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1674748b39499f8b46eb3b4580ed06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731147483751 2024-11-09T10:18:06,376 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b21f2c0a3e4642a3eeeeccfaeeb2d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147484390 2024-11-09T10:18:06,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092395030e43ed4649b27c695027baae07_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147485042/Put/seqid=0 2024-11-09T10:18:06,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742350_1526 (size=31058) 2024-11-09T10:18:06,408 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:06,409 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/44412fb044a74ec5836b7e4ca1991f1d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44412fb044a74ec5836b7e4ca1991f1d 2024-11-09T10:18:06,409 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/31eb3bb861464c39808d1b5f6c17ec25 is 50, key is test_row_0/C:col10/1731147485030/Put/seqid=0 2024-11-09T10:18:06,414 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into 44412fb044a74ec5836b7e4ca1991f1d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:06,414 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:06,414 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=13, startTime=1731147486291; duration=0sec 2024-11-09T10:18:06,414 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:06,414 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:06,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742351_1527 (size=12154) 2024-11-09T10:18:06,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742352_1528 (size=12104) 2024-11-09T10:18:06,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092395030e43ed4649b27c695027baae07_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092395030e43ed4649b27c695027baae07_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c3f7bf92143440a694bb6f95c3bd8e2d, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c3f7bf92143440a694bb6f95c3bd8e2d is 175, key is test_row_0/A:col10/1731147485042/Put/seqid=0 2024-11-09T10:18:06,432 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/31eb3bb861464c39808d1b5f6c17ec25 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/31eb3bb861464c39808d1b5f6c17ec25 2024-11-09T10:18:06,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742353_1529 (size=30955) 2024-11-09T10:18:06,441 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c3f7bf92143440a694bb6f95c3bd8e2d 2024-11-09T10:18:06,443 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into 31eb3bb861464c39808d1b5f6c17ec25(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:06,443 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:06,443 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=13, startTime=1731147486291; duration=0sec 2024-11-09T10:18:06,443 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:06,443 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:06,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/347f7aca24324496a3aa921de6b4832c is 50, key is test_row_0/B:col10/1731147485042/Put/seqid=0 2024-11-09T10:18:06,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742354_1530 (size=12001) 2024-11-09T10:18:06,468 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/347f7aca24324496a3aa921de6b4832c 2024-11-09T10:18:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/b8d465a690554c4ba37bf637555f6399 is 50, key is test_row_0/C:col10/1731147485042/Put/seqid=0 2024-11-09T10:18:06,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742355_1531 (size=12001) 2024-11-09T10:18:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T10:18:06,887 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/b8d465a690554c4ba37bf637555f6399 2024-11-09T10:18:06,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c3f7bf92143440a694bb6f95c3bd8e2d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c3f7bf92143440a694bb6f95c3bd8e2d 2024-11-09T10:18:06,894 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c3f7bf92143440a694bb6f95c3bd8e2d, entries=150, sequenceid=68, filesize=30.2 K 2024-11-09T10:18:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/347f7aca24324496a3aa921de6b4832c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/347f7aca24324496a3aa921de6b4832c 2024-11-09T10:18:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,897 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/347f7aca24324496a3aa921de6b4832c, entries=150, sequenceid=68, filesize=11.7 K 2024-11-09T10:18:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/b8d465a690554c4ba37bf637555f6399 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/b8d465a690554c4ba37bf637555f6399 2024-11-09T10:18:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,901 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/b8d465a690554c4ba37bf637555f6399, entries=150, sequenceid=68, filesize=11.7 K 2024-11-09T10:18:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,901 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for be37ddd6456db463b3a7fa388151b50e in 546ms, sequenceid=68, compaction requested=false 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-09T10:18:06,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.0100 sec 2024-11-09T10:18:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T10:18:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,039 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-09T10:18:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T10:18:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,042 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,042 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:07,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T10:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T10:18:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-09T10:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-09T10:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-09T10:18:07,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-09T10:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 159 msec 2024-11-09T10:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:07,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:07,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e9880089672745219d35b7c45173255e_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147487226/Put/seqid=0 2024-11-09T10:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742357_1533 (size=24358) 2024-11-09T10:18:07,267 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:07,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e9880089672745219d35b7c45173255e_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e9880089672745219d35b7c45173255e_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:07,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a80a780ed0f84adc941272a6cad8690c, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:07,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a80a780ed0f84adc941272a6cad8690c is 175, key is test_row_0/A:col10/1731147487226/Put/seqid=0 2024-11-09T10:18:07,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147547267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147547268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147547270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147487258, max timestamp=1731147487275Num uniques 859; 2024-11-09T10:18:07,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:18:07,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147487258, max timestamp=1731147487275Num uniques 859; 2024-11-09T10:18:07,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:18:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731147487258, max timestamp=1731147487275Num uniques 859; 2024-11-09T10:18:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:18:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742356_1532 (size=73995) 2024-11-09T10:18:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147547323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147547323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T10:18:07,358 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:07,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-09T10:18:07,360 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:07,361 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:07,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T10:18:07,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147547389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147547389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147547389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147547438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147547438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T10:18:07,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147547599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147547599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147547599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147547649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147547649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:07,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:07,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T10:18:07,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a80a780ed0f84adc941272a6cad8690c 2024-11-09T10:18:07,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/b2c82d3b7c0a4bb68d6636ea2708e8c5 is 50, key is test_row_0/B:col10/1731147487226/Put/seqid=0 2024-11-09T10:18:07,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742358_1534 (size=12001) 2024-11-09T10:18:07,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147547909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147547909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147547909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147547958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147547958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:07,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:07,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T10:18:08,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/b2c82d3b7c0a4bb68d6636ea2708e8c5 2024-11-09T10:18:08,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c9991d57696f45e3a32ef44d6e680ba1 is 50, key is test_row_0/C:col10/1731147487226/Put/seqid=0 2024-11-09T10:18:08,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742359_1535 (size=12001) 2024-11-09T10:18:08,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147548418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147548419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147548419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:08,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:08,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:08,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147548469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147548469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T10:18:08,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c9991d57696f45e3a32ef44d6e680ba1 2024-11-09T10:18:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a80a780ed0f84adc941272a6cad8690c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a80a780ed0f84adc941272a6cad8690c 2024-11-09T10:18:08,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a80a780ed0f84adc941272a6cad8690c, entries=400, sequenceid=83, filesize=72.3 K 2024-11-09T10:18:08,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/b2c82d3b7c0a4bb68d6636ea2708e8c5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/b2c82d3b7c0a4bb68d6636ea2708e8c5 2024-11-09T10:18:08,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/b2c82d3b7c0a4bb68d6636ea2708e8c5, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T10:18:08,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c9991d57696f45e3a32ef44d6e680ba1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c9991d57696f45e3a32ef44d6e680ba1 2024-11-09T10:18:08,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c9991d57696f45e3a32ef44d6e680ba1, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T10:18:08,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for be37ddd6456db463b3a7fa388151b50e in 1289ms, sequenceid=83, compaction requested=true 2024-11-09T10:18:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:08,525 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:08,525 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:08,526 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136008 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:08,526 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:08,526 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,526 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44412fb044a74ec5836b7e4ca1991f1d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c3f7bf92143440a694bb6f95c3bd8e2d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a80a780ed0f84adc941272a6cad8690c] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=132.8 K 2024-11-09T10:18:08,526 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,526 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44412fb044a74ec5836b7e4ca1991f1d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c3f7bf92143440a694bb6f95c3bd8e2d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a80a780ed0f84adc941272a6cad8690c] 2024-11-09T10:18:08,526 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:08,526 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:08,526 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,526 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/78fd1dd6f2314e468d8e8aef172ed50c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/347f7aca24324496a3aa921de6b4832c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/b2c82d3b7c0a4bb68d6636ea2708e8c5] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=35.3 K 2024-11-09T10:18:08,527 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44412fb044a74ec5836b7e4ca1991f1d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147484390 2024-11-09T10:18:08,527 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 78fd1dd6f2314e468d8e8aef172ed50c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147484390 2024-11-09T10:18:08,527 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f7bf92143440a694bb6f95c3bd8e2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731147485038 2024-11-09T10:18:08,527 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 347f7aca24324496a3aa921de6b4832c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731147485038 2024-11-09T10:18:08,527 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80a780ed0f84adc941272a6cad8690c, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731147487226 2024-11-09T10:18:08,527 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c82d3b7c0a4bb68d6636ea2708e8c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731147487226 2024-11-09T10:18:08,541 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:08,542 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/dfde14536f15484398485bd2ec8296a6 is 50, key is test_row_0/B:col10/1731147487226/Put/seqid=0 2024-11-09T10:18:08,546 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:08,564 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109b5a2d02ef2f24e2895b87e979581001d_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:08,568 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109b5a2d02ef2f24e2895b87e979581001d_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:08,568 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b5a2d02ef2f24e2895b87e979581001d_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:08,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T10:18:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,581 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-09T10:18:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:08,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742360_1536 (size=12207) 2024-11-09T10:18:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:08,587 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/dfde14536f15484398485bd2ec8296a6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/dfde14536f15484398485bd2ec8296a6 2024-11-09T10:18:08,593 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into dfde14536f15484398485bd2ec8296a6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:08,593 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:08,593 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=13, startTime=1731147488525; duration=0sec 2024-11-09T10:18:08,593 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:08,593 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:08,593 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:08,594 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:08,594 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:08,594 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:08,594 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/31eb3bb861464c39808d1b5f6c17ec25, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/b8d465a690554c4ba37bf637555f6399, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c9991d57696f45e3a32ef44d6e680ba1] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=35.3 K 2024-11-09T10:18:08,594 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 31eb3bb861464c39808d1b5f6c17ec25, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731147484390 2024-11-09T10:18:08,594 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d465a690554c4ba37bf637555f6399, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731147485038 2024-11-09T10:18:08,595 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting c9991d57696f45e3a32ef44d6e680ba1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731147487226 2024-11-09T10:18:08,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742361_1537 (size=4469) 2024-11-09T10:18:08,603 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:08,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109260b63e6e86d4fad871778e56a406bde_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147487322/Put/seqid=0 2024-11-09T10:18:08,603 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/46b7b781f22b45a4a85ba3cce5ac0c5c is 50, key is test_row_0/C:col10/1731147487226/Put/seqid=0 2024-11-09T10:18:08,610 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#452 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:08,610 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/1760682999604cb282382fb316653d56 is 175, key is test_row_0/A:col10/1731147487226/Put/seqid=0 2024-11-09T10:18:08,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742362_1538 (size=12207) 2024-11-09T10:18:08,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742364_1540 (size=31161) 2024-11-09T10:18:08,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742363_1539 (size=12154) 2024-11-09T10:18:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:08,685 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/46b7b781f22b45a4a85ba3cce5ac0c5c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/46b7b781f22b45a4a85ba3cce5ac0c5c 2024-11-09T10:18:08,689 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109260b63e6e86d4fad871778e56a406bde_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109260b63e6e86d4fad871778e56a406bde_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:08,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c4078915945b48af85377d44abb1c213, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:08,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c4078915945b48af85377d44abb1c213 is 175, key is test_row_0/A:col10/1731147487322/Put/seqid=0 2024-11-09T10:18:08,693 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into 46b7b781f22b45a4a85ba3cce5ac0c5c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:08,693 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:08,693 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=13, startTime=1731147488525; duration=0sec 2024-11-09T10:18:08,693 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:08,693 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:08,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742365_1541 (size=30955) 2024-11-09T10:18:08,709 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c4078915945b48af85377d44abb1c213 2024-11-09T10:18:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/5e82549f4cdc4521ab7c44130aa58358 is 50, key is test_row_0/B:col10/1731147487322/Put/seqid=0 2024-11-09T10:18:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742366_1542 (size=12001) 2024-11-09T10:18:09,084 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/1760682999604cb282382fb316653d56 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1760682999604cb282382fb316653d56 2024-11-09T10:18:09,089 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into 1760682999604cb282382fb316653d56(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:09,089 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:09,089 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=13, startTime=1731147488525; duration=0sec 2024-11-09T10:18:09,089 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:09,089 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:09,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/5e82549f4cdc4521ab7c44130aa58358 2024-11-09T10:18:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/57f29e7b3de34a4c9a0305efe1319abd is 50, key is test_row_0/C:col10/1731147487322/Put/seqid=0 2024-11-09T10:18:09,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742367_1543 (size=12001) 2024-11-09T10:18:09,201 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/57f29e7b3de34a4c9a0305efe1319abd 2024-11-09T10:18:09,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c4078915945b48af85377d44abb1c213 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c4078915945b48af85377d44abb1c213 2024-11-09T10:18:09,213 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c4078915945b48af85377d44abb1c213, entries=150, sequenceid=118, filesize=30.2 K 2024-11-09T10:18:09,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/5e82549f4cdc4521ab7c44130aa58358 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/5e82549f4cdc4521ab7c44130aa58358 2024-11-09T10:18:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,217 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/5e82549f4cdc4521ab7c44130aa58358, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T10:18:09,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/57f29e7b3de34a4c9a0305efe1319abd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/57f29e7b3de34a4c9a0305efe1319abd 2024-11-09T10:18:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/57f29e7b3de34a4c9a0305efe1319abd, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T10:18:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,223 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for be37ddd6456db463b3a7fa388151b50e in 641ms, sequenceid=118, compaction requested=false 2024-11-09T10:18:09,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:09,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-09T10:18:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-09T10:18:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-09T10:18:09,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-11-09T10:18:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8670 sec 2024-11-09T10:18:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:09,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T10:18:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096f807253bf304228a66179f9832f967d_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147489475/Put/seqid=0 2024-11-09T10:18:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742368_1544 (size=19624) 2024-11-09T10:18:09,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096f807253bf304228a66179f9832f967d_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096f807253bf304228a66179f9832f967d_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:09,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/49d8e21353b34c9993dd3a6a8a949572, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:09,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/49d8e21353b34c9993dd3a6a8a949572 is 175, key is test_row_0/A:col10/1731147489475/Put/seqid=0 2024-11-09T10:18:09,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742369_1545 (size=56883) 2024-11-09T10:18:09,515 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/49d8e21353b34c9993dd3a6a8a949572 2024-11-09T10:18:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T10:18:09,519 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:09,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-09T10:18:09,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T10:18:09,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:09,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:09,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/af80eda4b8b94e648b39c4b19e514545 is 50, key is test_row_0/B:col10/1731147489475/Put/seqid=0 2024-11-09T10:18:09,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147489507, max timestamp=1731147489522Num uniques 859; 2024-11-09T10:18:09,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:18:09,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147489507, max timestamp=1731147489522Num uniques 859; 2024-11-09T10:18:09,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:18:09,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147489507, max timestamp=1731147489522Num uniques 859; 2024-11-09T10:18:09,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:18:09,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742370_1546 (size=12151) 2024-11-09T10:18:09,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/af80eda4b8b94e648b39c4b19e514545 2024-11-09T10:18:09,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/71edd54fad29451c87c4f05e3b121537 is 50, key is test_row_0/C:col10/1731147489475/Put/seqid=0 2024-11-09T10:18:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147549547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742371_1547 (size=12151) 2024-11-09T10:18:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147549548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147549549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/71edd54fad29451c87c4f05e3b121537 2024-11-09T10:18:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/49d8e21353b34c9993dd3a6a8a949572 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/49d8e21353b34c9993dd3a6a8a949572 2024-11-09T10:18:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/49d8e21353b34c9993dd3a6a8a949572, entries=300, sequenceid=131, filesize=55.5 K 2024-11-09T10:18:09,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/af80eda4b8b94e648b39c4b19e514545 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/af80eda4b8b94e648b39c4b19e514545 2024-11-09T10:18:09,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147549556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147549556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/af80eda4b8b94e648b39c4b19e514545, entries=150, sequenceid=131, filesize=11.9 K 2024-11-09T10:18:09,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/71edd54fad29451c87c4f05e3b121537 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/71edd54fad29451c87c4f05e3b121537 2024-11-09T10:18:09,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/71edd54fad29451c87c4f05e3b121537, entries=150, sequenceid=131, filesize=11.9 K 2024-11-09T10:18:09,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for be37ddd6456db463b3a7fa388151b50e in 90ms, sequenceid=131, compaction requested=true 2024-11-09T10:18:09,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:09,572 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:09,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:09,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:09,573 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:09,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:09,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:09,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:09,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:09,574 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:09,574 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:09,575 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,575 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1760682999604cb282382fb316653d56, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c4078915945b48af85377d44abb1c213, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/49d8e21353b34c9993dd3a6a8a949572] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=116.2 K 2024-11-09T10:18:09,575 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,575 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1760682999604cb282382fb316653d56, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c4078915945b48af85377d44abb1c213, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/49d8e21353b34c9993dd3a6a8a949572] 2024-11-09T10:18:09,575 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:09,575 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:09,575 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,575 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/dfde14536f15484398485bd2ec8296a6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/5e82549f4cdc4521ab7c44130aa58358, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/af80eda4b8b94e648b39c4b19e514545] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=35.5 K 2024-11-09T10:18:09,576 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting dfde14536f15484398485bd2ec8296a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731147487226 2024-11-09T10:18:09,577 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e82549f4cdc4521ab7c44130aa58358, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731147487315 2024-11-09T10:18:09,577 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1760682999604cb282382fb316653d56, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731147487226 2024-11-09T10:18:09,577 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting af80eda4b8b94e648b39c4b19e514545, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147489471 2024-11-09T10:18:09,577 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4078915945b48af85377d44abb1c213, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731147487315 2024-11-09T10:18:09,578 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49d8e21353b34c9993dd3a6a8a949572, keycount=300, bloomtype=ROW, size=55.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147489458 2024-11-09T10:18:09,586 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:09,587 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/a8e2d2be802b4658b19c5deef8021a91 is 50, key is test_row_0/B:col10/1731147489475/Put/seqid=0 2024-11-09T10:18:09,589 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:09,613 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109919ade44eaeb491cabcc38b795639b7e_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:09,615 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109919ade44eaeb491cabcc38b795639b7e_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:09,615 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109919ade44eaeb491cabcc38b795639b7e_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T10:18:09,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742372_1548 (size=12459) 2024-11-09T10:18:09,642 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/a8e2d2be802b4658b19c5deef8021a91 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a8e2d2be802b4658b19c5deef8021a91 2024-11-09T10:18:09,647 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into a8e2d2be802b4658b19c5deef8021a91(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:09,647 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:09,647 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=13, startTime=1731147489573; duration=0sec 2024-11-09T10:18:09,647 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:09,647 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:09,647 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:09,649 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:09,649 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:09,649 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,649 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/46b7b781f22b45a4a85ba3cce5ac0c5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/57f29e7b3de34a4c9a0305efe1319abd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/71edd54fad29451c87c4f05e3b121537] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=35.5 K 2024-11-09T10:18:09,650 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 46b7b781f22b45a4a85ba3cce5ac0c5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731147487226 2024-11-09T10:18:09,650 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f29e7b3de34a4c9a0305efe1319abd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731147487315 2024-11-09T10:18:09,650 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 71edd54fad29451c87c4f05e3b121537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147489471 2024-11-09T10:18:09,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742373_1549 (size=4469) 2024-11-09T10:18:09,659 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:09,660 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/8f778a343f88417480e0af14a0bffb04 is 50, key is test_row_0/C:col10/1731147489475/Put/seqid=0 2024-11-09T10:18:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:09,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-09T10:18:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:18:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:18:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:18:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147549669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147549670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147549670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T10:18:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147549673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147549674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742374_1550 (size=12459) 2024-11-09T10:18:09,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094cf015777ed74742a77a09077170e3ec_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147489554/Put/seqid=0 2024-11-09T10:18:09,691 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/8f778a343f88417480e0af14a0bffb04 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8f778a343f88417480e0af14a0bffb04 2024-11-09T10:18:09,696 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into 8f778a343f88417480e0af14a0bffb04(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:09,696 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:09,696 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=13, startTime=1731147489573; duration=0sec 2024-11-09T10:18:09,696 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:09,696 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:09,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742375_1551 (size=14794) 2024-11-09T10:18:09,712 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:09,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094cf015777ed74742a77a09077170e3ec_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094cf015777ed74742a77a09077170e3ec_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:09,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/f1e0827456154986afc8898f6c6ec929, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:09,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/f1e0827456154986afc8898f6c6ec929 is 175, key is test_row_0/A:col10/1731147489554/Put/seqid=0 2024-11-09T10:18:09,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742376_1552 (size=39749) 2024-11-09T10:18:09,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/f1e0827456154986afc8898f6c6ec929 2024-11-09T10:18:09,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/7b34129a600d4b48b3b632f1140fd7c7 is 50, key is test_row_0/B:col10/1731147489554/Put/seqid=0 2024-11-09T10:18:09,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742377_1553 (size=12151) 2024-11-09T10:18:09,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/7b34129a600d4b48b3b632f1140fd7c7 2024-11-09T10:18:09,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/e65ad687d0024fec97c97fb6c3777e5f is 50, key is test_row_0/C:col10/1731147489554/Put/seqid=0 2024-11-09T10:18:09,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147549779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742378_1554 (size=12151) 2024-11-09T10:18:09,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/e65ad687d0024fec97c97fb6c3777e5f 2024-11-09T10:18:09,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147549788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147549789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:09,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/f1e0827456154986afc8898f6c6ec929 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/f1e0827456154986afc8898f6c6ec929 2024-11-09T10:18:09,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/f1e0827456154986afc8898f6c6ec929, entries=200, sequenceid=170, filesize=38.8 K 2024-11-09T10:18:09,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/7b34129a600d4b48b3b632f1140fd7c7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/7b34129a600d4b48b3b632f1140fd7c7 2024-11-09T10:18:09,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/7b34129a600d4b48b3b632f1140fd7c7, entries=150, sequenceid=170, filesize=11.9 K 2024-11-09T10:18:09,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/e65ad687d0024fec97c97fb6c3777e5f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/e65ad687d0024fec97c97fb6c3777e5f 2024-11-09T10:18:09,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/e65ad687d0024fec97c97fb6c3777e5f, entries=150, sequenceid=170, filesize=11.9 K 2024-11-09T10:18:09,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for be37ddd6456db463b3a7fa388151b50e in 164ms, sequenceid=170, compaction requested=false 2024-11-09T10:18:09,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:09,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T10:18:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:09,831 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-09T10:18:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T10:18:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090d2dd079100d4506800fcc1a3a9422e0_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147489666/Put/seqid=0 2024-11-09T10:18:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:09,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742379_1555 (size=12304) 2024-11-09T10:18:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147489944, max timestamp=1731147489979Num uniques 859; 2024-11-09T10:18:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147489944, max timestamp=1731147489979Num uniques 859; 2024-11-09T10:18:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:18:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:18:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731147489944, max timestamp=1731147489979Num uniques 859; 2024-11-09T10:18:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T10:18:10,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147550005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147550007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147550008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147550010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147550011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,055 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#461 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:10,056 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/6945a7e2f9e6429288a5f2f019ef31c2 is 175, key is test_row_0/A:col10/1731147489475/Put/seqid=0 2024-11-09T10:18:10,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742380_1556 (size=31413) 2024-11-09T10:18:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147550119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147550119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147550119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147550128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147550129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T10:18:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:10,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090d2dd079100d4506800fcc1a3a9422e0_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d2dd079100d4506800fcc1a3a9422e0_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:10,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/48e828953f0c495b8df5880eef3927e6, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:10,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/48e828953f0c495b8df5880eef3927e6 is 175, key is test_row_0/A:col10/1731147489666/Put/seqid=0 2024-11-09T10:18:10,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742381_1557 (size=31105) 2024-11-09T10:18:10,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/48e828953f0c495b8df5880eef3927e6 2024-11-09T10:18:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/e06468637b0e47d0bdd27844262a7fec is 50, key is test_row_0/B:col10/1731147489666/Put/seqid=0 2024-11-09T10:18:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147550329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147550330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147550330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147550339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147550339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742382_1558 (size=12151) 2024-11-09T10:18:10,348 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/e06468637b0e47d0bdd27844262a7fec 2024-11-09T10:18:10,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/a431018662244275a39cda8c258b2f2d is 50, key is test_row_0/C:col10/1731147489666/Put/seqid=0 2024-11-09T10:18:10,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742383_1559 (size=12151) 2024-11-09T10:18:10,403 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/a431018662244275a39cda8c258b2f2d 2024-11-09T10:18:10,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/48e828953f0c495b8df5880eef3927e6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/48e828953f0c495b8df5880eef3927e6 2024-11-09T10:18:10,423 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/48e828953f0c495b8df5880eef3927e6, entries=150, sequenceid=180, filesize=30.4 K 2024-11-09T10:18:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/e06468637b0e47d0bdd27844262a7fec as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e06468637b0e47d0bdd27844262a7fec 2024-11-09T10:18:10,428 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e06468637b0e47d0bdd27844262a7fec, entries=150, sequenceid=180, filesize=11.9 K 2024-11-09T10:18:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/a431018662244275a39cda8c258b2f2d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/a431018662244275a39cda8c258b2f2d 2024-11-09T10:18:10,433 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/a431018662244275a39cda8c258b2f2d, entries=150, sequenceid=180, filesize=11.9 K 2024-11-09T10:18:10,434 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for be37ddd6456db463b3a7fa388151b50e in 603ms, sequenceid=180, compaction requested=true 2024-11-09T10:18:10,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:10,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:10,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-09T10:18:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-09T10:18:10,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-09T10:18:10,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 913 msec 2024-11-09T10:18:10,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 920 msec 2024-11-09T10:18:10,463 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/6945a7e2f9e6429288a5f2f019ef31c2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6945a7e2f9e6429288a5f2f019ef31c2 2024-11-09T10:18:10,469 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into 6945a7e2f9e6429288a5f2f019ef31c2(size=30.7 K), total size for store is 99.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:10,469 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:10,470 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=13, startTime=1731147489572; duration=0sec 2024-11-09T10:18:10,470 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:10,470 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147489998, max timestamp=1731147490641Num uniques 1068; 2024-11-09T10:18:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147489998, max timestamp=1731147490641Num uniques 1068; 2024-11-09T10:18:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T10:18:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T10:18:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147489998, max timestamp=1731147490641Num uniques 1068; 2024-11-09T10:18:10,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=67.09 KB heapSize=183.88 KB 2024-11-09T10:18:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T10:18:10,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:10,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:10,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:10,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:10,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:10,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T10:18:10,659 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:10,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-09T10:18:10,661 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T10:18:10,662 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:10,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:10,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f31a071359214b63b6349e96113fb7f5_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147490644/Put/seqid=0 2024-11-09T10:18:10,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147490670, max timestamp=1731147490679Num uniques 859; 2024-11-09T10:18:10,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731147490670, max timestamp=1731147490679Num uniques 859; 2024-11-09T10:18:10,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:18:10,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T10:18:10,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1731147490676, max timestamp=1731147490679Num uniques 859; 2024-11-09T10:18:10,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-09T10:18:10,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742384_1560 (size=19774) 2024-11-09T10:18:10,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:10,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f31a071359214b63b6349e96113fb7f5_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f31a071359214b63b6349e96113fb7f5_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:10,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/4a6f9cdd48cf4d04a39632647c9345b1, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:10,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/4a6f9cdd48cf4d04a39632647c9345b1 is 175, key is test_row_0/A:col10/1731147490644/Put/seqid=0 2024-11-09T10:18:10,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742385_1561 (size=57033) 2024-11-09T10:18:10,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/4a6f9cdd48cf4d04a39632647c9345b1 2024-11-09T10:18:10,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/e4410e5d6bc04c39945af4defa557eca is 50, key is test_row_0/B:col10/1731147490644/Put/seqid=0 2024-11-09T10:18:10,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742386_1562 (size=12151) 2024-11-09T10:18:10,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147550735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147550735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147550736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147550736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147490704, max timestamp=1731147490743Num uniques 1118; 2024-11-09T10:18:10,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147490704, max timestamp=1731147490743Num uniques 1118; 2024-11-09T10:18:10,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147490704, max timestamp=1731147490743Num uniques 968; 2024-11-09T10:18:10,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-09T10:18:10,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-09T10:18:10,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-11-09T10:18:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T10:18:10,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:10,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147550859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147550859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10387, totalHeapSize=23848, min timestamp=1731147490825, max timestamp=1731147490864Num uniques 1027; 2024-11-09T10:18:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1731147490825, max timestamp=1731147490864Num uniques 1027; 2024-11-09T10:18:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47027, new segment data size=10387, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-09T10:18:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47022, new segment data size=10382, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-09T10:18:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1731147490825, max timestamp=1731147490864Num uniques 1027; 2024-11-09T10:18:10,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47022, new segment data size=10382, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-09T10:18:10,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147550905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147550906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:10,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731147490896, max timestamp=1731147490912Num uniques 1086; 2024-11-09T10:18:10,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731147490896, max timestamp=1731147490912Num uniques 1086; 2024-11-09T10:18:10,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-09T10:18:10,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-09T10:18:10,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731147490896, max timestamp=1731147490912Num uniques 1086; 2024-11-09T10:18:10,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-11-09T10:18:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:10,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:10,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:10,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:10,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:10,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:10,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T10:18:11,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731147490995, max timestamp=1731147491026Num uniques 1045; 2024-11-09T10:18:11,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-09T10:18:11,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731147490995, max timestamp=1731147491026Num uniques 1045; 2024-11-09T10:18:11,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731147490995, max timestamp=1731147491026Num uniques 1009; 2024-11-09T10:18:11,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-09T10:18:11,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-11-09T10:18:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147551070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147551071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731147491068, max timestamp=1731147491075Num uniques 1054; 2024-11-09T10:18:11,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731147491068, max timestamp=1731147491075Num uniques 1054; 2024-11-09T10:18:11,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-09T10:18:11,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-09T10:18:11,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731147491068, max timestamp=1731147491075Num uniques 1054; 2024-11-09T10:18:11,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-09T10:18:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:11,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:11,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/e4410e5d6bc04c39945af4defa557eca 2024-11-09T10:18:11,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731147491095, max timestamp=1731147491127Num uniques 1013; 2024-11-09T10:18:11,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T10:18:11,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/83cbbe84d51d46d7ad1b4b422c2af194 is 50, key is test_row_0/C:col10/1731147490644/Put/seqid=0 2024-11-09T10:18:11,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731147491095, max timestamp=1731147491130Num uniques 1013; 2024-11-09T10:18:11,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T10:18:11,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731147491095, max timestamp=1731147491130Num uniques 1013; 2024-11-09T10:18:11,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T10:18:11,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742387_1563 (size=12151) 2024-11-09T10:18:11,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731147491163, max timestamp=1731147491174Num uniques 1072; 2024-11-09T10:18:11,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731147491163, max timestamp=1731147491174Num uniques 1072; 2024-11-09T10:18:11,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-09T10:18:11,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-09T10:18:11,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731147491164, max timestamp=1731147491174Num uniques 1072; 2024-11-09T10:18:11,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-09T10:18:11,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147551201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147551204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731147491189, max timestamp=1731147491208Num uniques 1131; 2024-11-09T10:18:11,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-09T10:18:11,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731147491189, max timestamp=1731147491208Num uniques 1131; 2024-11-09T10:18:11,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-09T10:18:11,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731147491189, max timestamp=1731147491208Num uniques 1031; 2024-11-09T10:18:11,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-09T10:18:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147551251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731147491246, max timestamp=1731147491255Num uniques 1090; 2024-11-09T10:18:11,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-09T10:18:11,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731147491246, max timestamp=1731147491255Num uniques 1090; 2024-11-09T10:18:11,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-09T10:18:11,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731147491246, max timestamp=1731147491255Num uniques 1090; 2024-11-09T10:18:11,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-09T10:18:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T10:18:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147551319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11394, totalHeapSize=26136, min timestamp=1731147491302, max timestamp=1731147491328Num uniques 1099; 2024-11-09T10:18:11,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731147491302, max timestamp=1731147491328Num uniques 1099; 2024-11-09T10:18:11,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=11394, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-09T10:18:11,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-09T10:18:11,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731147491303, max timestamp=1731147491328Num uniques 1099; 2024-11-09T10:18:11,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-09T10:18:11,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147551379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731147491368, max timestamp=1731147491387Num uniques 1108; 2024-11-09T10:18:11,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-09T10:18:11,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731147491368, max timestamp=1731147491387Num uniques 1108; 2024-11-09T10:18:11,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-09T10:18:11,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731147491368, max timestamp=1731147491387Num uniques 1058; 2024-11-09T10:18:11,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48448, new segment data size=9518, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-09T10:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:11,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:11,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147551433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147551435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731147491408, max timestamp=1731147491438Num uniques 1067; 2024-11-09T10:18:11,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-09T10:18:11,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731147491408, max timestamp=1731147491438Num uniques 1067; 2024-11-09T10:18:11,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-09T10:18:11,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731147491408, max timestamp=1731147491438Num uniques 1067; 2024-11-09T10:18:11,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-09T10:18:11,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731147491503, max timestamp=1731147491519Num uniques 1026; 2024-11-09T10:18:11,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-09T10:18:11,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731147491503, max timestamp=1731147491519Num uniques 1026; 2024-11-09T10:18:11,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-09T10:18:11,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731147491503, max timestamp=1731147491519Num uniques 1026; 2024-11-09T10:18:11,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-09T10:18:11,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/83cbbe84d51d46d7ad1b4b422c2af194 2024-11-09T10:18:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/4a6f9cdd48cf4d04a39632647c9345b1 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/4a6f9cdd48cf4d04a39632647c9345b1 2024-11-09T10:18:11,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/4a6f9cdd48cf4d04a39632647c9345b1, entries=300, sequenceid=221, filesize=55.7 K 2024-11-09T10:18:11,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/e4410e5d6bc04c39945af4defa557eca as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e4410e5d6bc04c39945af4defa557eca 2024-11-09T10:18:11,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e4410e5d6bc04c39945af4defa557eca, entries=150, sequenceid=221, filesize=11.9 K 2024-11-09T10:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/83cbbe84d51d46d7ad1b4b422c2af194 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/83cbbe84d51d46d7ad1b4b422c2af194 2024-11-09T10:18:11,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/83cbbe84d51d46d7ad1b4b422c2af194, entries=150, sequenceid=221, filesize=11.9 K 2024-11-09T10:18:11,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=140.89 KB/144270 for be37ddd6456db463b3a7fa388151b50e in 920ms, sequenceid=221, compaction requested=true 2024-11-09T10:18:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:11,566 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:11,566 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:11,567 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:11,567 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:11,567 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,567 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6945a7e2f9e6429288a5f2f019ef31c2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/f1e0827456154986afc8898f6c6ec929, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/48e828953f0c495b8df5880eef3927e6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/4a6f9cdd48cf4d04a39632647c9345b1] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=155.6 K 2024-11-09T10:18:11,567 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,567 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6945a7e2f9e6429288a5f2f019ef31c2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/f1e0827456154986afc8898f6c6ec929, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/48e828953f0c495b8df5880eef3927e6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/4a6f9cdd48cf4d04a39632647c9345b1] 2024-11-09T10:18:11,567 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:11,567 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:11,567 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,568 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a8e2d2be802b4658b19c5deef8021a91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/7b34129a600d4b48b3b632f1140fd7c7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e06468637b0e47d0bdd27844262a7fec, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e4410e5d6bc04c39945af4defa557eca] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=47.8 K 2024-11-09T10:18:11,568 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6945a7e2f9e6429288a5f2f019ef31c2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147489471 2024-11-09T10:18:11,568 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e2d2be802b4658b19c5deef8021a91, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147489471 2024-11-09T10:18:11,569 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1e0827456154986afc8898f6c6ec929, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731147489546 2024-11-09T10:18:11,569 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b34129a600d4b48b3b632f1140fd7c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731147489554 2024-11-09T10:18:11,569 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e828953f0c495b8df5880eef3927e6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731147489666 2024-11-09T10:18:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:11,569 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e06468637b0e47d0bdd27844262a7fec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731147489666 2024-11-09T10:18:11,569 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6f9cdd48cf4d04a39632647c9345b1, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147489998 2024-11-09T10:18:11,569 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e4410e5d6bc04c39945af4defa557eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147490641 2024-11-09T10:18:11,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=154.31 KB heapSize=439.17 KB 2024-11-09T10:18:11,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:11,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:18:11,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:11,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:18:11,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:11,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:18:11,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147551572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147551572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:11,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147551573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147551573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147551575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,586 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:11,586 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/cab9045c37fe4db3b03466f19f25d5d3 is 50, key is test_row_0/B:col10/1731147490644/Put/seqid=0 2024-11-09T10:18:11,588 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:11,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090f9b318f543648d4a59c123bc96d20c0_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147491569/Put/seqid=0 2024-11-09T10:18:11,603 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109f2fad39a8a4145fcba0479c0af1c9d8b_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:11,605 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109f2fad39a8a4145fcba0479c0af1c9d8b_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:11,605 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f2fad39a8a4145fcba0479c0af1c9d8b_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:11,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742388_1564 (size=12595) 2024-11-09T10:18:11,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742389_1565 (size=22618) 2024-11-09T10:18:11,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:11,620 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090f9b318f543648d4a59c123bc96d20c0_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090f9b318f543648d4a59c123bc96d20c0_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:11,621 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/44450056c21846ca923e0719c9cc4976, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:11,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/44450056c21846ca923e0719c9cc4976 is 175, key is test_row_0/A:col10/1731147491569/Put/seqid=0 2024-11-09T10:18:11,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742390_1566 (size=4469) 2024-11-09T10:18:11,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742391_1567 (size=66023) 2024-11-09T10:18:11,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=485, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/44450056c21846ca923e0719c9cc4976 2024-11-09T10:18:11,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/0686517d9003419f98032a9685d2742d is 50, key is test_row_0/B:col10/1731147491569/Put/seqid=0 2024-11-09T10:18:11,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742392_1568 (size=12301) 2024-11-09T10:18:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147551678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147551678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147551689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147551689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147551689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:11,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:11,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T10:18:11,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:11,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:11,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147551888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147551888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147551899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147551899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147551899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,019 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/cab9045c37fe4db3b03466f19f25d5d3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/cab9045c37fe4db3b03466f19f25d5d3 2024-11-09T10:18:12,022 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into cab9045c37fe4db3b03466f19f25d5d3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:12,022 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:12,022 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=12, startTime=1731147491566; duration=0sec 2024-11-09T10:18:12,022 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:12,022 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:12,022 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:12,024 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:12,024 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:12,024 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:12,024 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8f778a343f88417480e0af14a0bffb04, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/e65ad687d0024fec97c97fb6c3777e5f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/a431018662244275a39cda8c258b2f2d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/83cbbe84d51d46d7ad1b4b422c2af194] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=47.8 K 2024-11-09T10:18:12,025 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f778a343f88417480e0af14a0bffb04, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731147489471 2024-11-09T10:18:12,025 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting e65ad687d0024fec97c97fb6c3777e5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731147489554 2024-11-09T10:18:12,026 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a431018662244275a39cda8c258b2f2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731147489666 2024-11-09T10:18:12,026 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 83cbbe84d51d46d7ad1b4b422c2af194, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147490641 2024-11-09T10:18:12,027 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#473 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:12,028 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/6b62f2abe28a41ce950b3802ec6788f3 is 175, key is test_row_0/A:col10/1731147490644/Put/seqid=0 2024-11-09T10:18:12,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742393_1569 (size=31549) 2024-11-09T10:18:12,032 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:12,032 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/bef15c122333425ab232adecd7c6ba94 is 50, key is test_row_0/C:col10/1731147490644/Put/seqid=0 2024-11-09T10:18:12,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742394_1570 (size=12595) 2024-11-09T10:18:12,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:12,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:12,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:12,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:12,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:12,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:12,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/0686517d9003419f98032a9685d2742d 2024-11-09T10:18:12,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/defb378623754cdc8092439e5cb7f39e is 50, key is test_row_0/C:col10/1731147491569/Put/seqid=0 2024-11-09T10:18:12,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742395_1571 (size=12301) 2024-11-09T10:18:12,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/defb378623754cdc8092439e5cb7f39e 2024-11-09T10:18:12,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/44450056c21846ca923e0719c9cc4976 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44450056c21846ca923e0719c9cc4976 2024-11-09T10:18:12,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44450056c21846ca923e0719c9cc4976, entries=350, sequenceid=485, filesize=64.5 K 2024-11-09T10:18:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/0686517d9003419f98032a9685d2742d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/0686517d9003419f98032a9685d2742d 2024-11-09T10:18:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/0686517d9003419f98032a9685d2742d, entries=150, sequenceid=485, filesize=12.0 K 2024-11-09T10:18:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/defb378623754cdc8092439e5cb7f39e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/defb378623754cdc8092439e5cb7f39e 2024-11-09T10:18:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/defb378623754cdc8092439e5cb7f39e, entries=150, sequenceid=485, filesize=12.0 K 2024-11-09T10:18:12,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~478.97 KB/490464, currentSize=13.42 KB/13740 for be37ddd6456db463b3a7fa388151b50e in 500ms, sequenceid=485, compaction requested=false 2024-11-09T10:18:12,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T10:18:12,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,190 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-09T10:18:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110955760977765a49bbb3c51a7def156c53_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147491573/Put/seqid=0 2024-11-09T10:18:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:12,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742396_1572 (size=9914) 2024-11-09T10:18:12,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147492241, max timestamp=1731147492251Num uniques 859; 2024-11-09T10:18:12,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731147492241, max timestamp=1731147492251Num uniques 859; 2024-11-09T10:18:12,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:18:12,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T10:18:12,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147552305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147552306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147552306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147552307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147552311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147552419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147552419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147552419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147552419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147552428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,437 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/6b62f2abe28a41ce950b3802ec6788f3 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6b62f2abe28a41ce950b3802ec6788f3 2024-11-09T10:18:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,441 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/bef15c122333425ab232adecd7c6ba94 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bef15c122333425ab232adecd7c6ba94 2024-11-09T10:18:12,442 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into 6b62f2abe28a41ce950b3802ec6788f3(size=30.8 K), total size for store is 95.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:12,442 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:12,442 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=12, startTime=1731147491566; duration=0sec 2024-11-09T10:18:12,443 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:12,443 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:12,446 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into bef15c122333425ab232adecd7c6ba94(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:12,446 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:12,446 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=12, startTime=1731147491566; duration=0sec 2024-11-09T10:18:12,446 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:12,446 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:12,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147552629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147552629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,633 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110955760977765a49bbb3c51a7def156c53_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110955760977765a49bbb3c51a7def156c53_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147552629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147552629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/0aac00805e3d4676bdd9c8664adf9277, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/0aac00805e3d4676bdd9c8664adf9277 is 175, key is test_row_0/A:col10/1731147491573/Put/seqid=0 2024-11-09T10:18:12,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147552649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742397_1573 (size=22561) 2024-11-09T10:18:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T10:18:12,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147552939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147552939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147552939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147552939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147552959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:13,058 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/0aac00805e3d4676bdd9c8664adf9277 2024-11-09T10:18:13,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/3657361e0b3f4e6a85c840f8385f8a1d is 50, key is test_row_0/B:col10/1731147491573/Put/seqid=0 2024-11-09T10:18:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742398_1574 (size=9857) 2024-11-09T10:18:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147553449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147553449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147553449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147553449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:13,470 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/3657361e0b3f4e6a85c840f8385f8a1d 2024-11-09T10:18:13,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147553469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/f542b2461dd1481f9e71df17fb53a98c is 50, key is test_row_0/C:col10/1731147491573/Put/seqid=0 2024-11-09T10:18:13,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742399_1575 (size=9857) 2024-11-09T10:18:13,877 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/f542b2461dd1481f9e71df17fb53a98c 2024-11-09T10:18:13,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/0aac00805e3d4676bdd9c8664adf9277 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/0aac00805e3d4676bdd9c8664adf9277 2024-11-09T10:18:13,883 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/0aac00805e3d4676bdd9c8664adf9277, entries=100, sequenceid=491, filesize=22.0 K 2024-11-09T10:18:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/3657361e0b3f4e6a85c840f8385f8a1d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3657361e0b3f4e6a85c840f8385f8a1d 2024-11-09T10:18:13,887 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3657361e0b3f4e6a85c840f8385f8a1d, entries=100, sequenceid=491, filesize=9.6 K 2024-11-09T10:18:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/f542b2461dd1481f9e71df17fb53a98c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/f542b2461dd1481f9e71df17fb53a98c 2024-11-09T10:18:13,891 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/f542b2461dd1481f9e71df17fb53a98c, entries=100, sequenceid=491, filesize=9.6 K 2024-11-09T10:18:13,892 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=165.49 KB/169460 for be37ddd6456db463b3a7fa388151b50e in 1701ms, sequenceid=491, compaction requested=true 2024-11-09T10:18:13,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:13,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:13,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-09T10:18:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-09T10:18:13,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-09T10:18:13,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2310 sec 2024-11-09T10:18:13,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.2350 sec 2024-11-09T10:18:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:14,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=172.20 KB heapSize=503.55 KB 2024-11-09T10:18:14,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147492304, max timestamp=1731147494463Num uniques 1018; 2024-11-09T10:18:14,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T10:18:14,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731147492304, max timestamp=1731147494463Num uniques 1018; 2024-11-09T10:18:14,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e067b2c2c7b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T10:18:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T10:18:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T10:18:14,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b5e7e579ae29424883615f4e6a529c7b_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147494464/Put/seqid=0 2024-11-09T10:18:14,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147554482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147554484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147554489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147554489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147554489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,497 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in be37ddd6456db463b3a7fa388151b50e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:18:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742400_1576 (size=14994) 2024-11-09T10:18:14,501 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:14,503 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b5e7e579ae29424883615f4e6a529c7b_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b5e7e579ae29424883615f4e6a529c7b_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:14,504 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/1576af48130742d095e2f7902afb7ac7, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:14,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/1576af48130742d095e2f7902afb7ac7 is 175, key is test_row_0/A:col10/1731147494464/Put/seqid=0 2024-11-09T10:18:14,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742401_1577 (size=39949) 2024-11-09T10:18:14,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147554599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147554599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147554599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147554608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147554809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147554809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147554809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147554819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T10:18:14,839 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:14,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-09T10:18:14,841 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T10:18:14,841 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:14,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:14,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=532, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/1576af48130742d095e2f7902afb7ac7 2024-11-09T10:18:14,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/20f7b7012dfb4cd89237ab9d2aaa4e7c is 50, key is test_row_0/B:col10/1731147494464/Put/seqid=0 2024-11-09T10:18:14,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742402_1578 (size=12301) 2024-11-09T10:18:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T10:18:14,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T10:18:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:14,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:15,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147555119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147555119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147555119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147555129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T10:18:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T10:18:15,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T10:18:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:15,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/20f7b7012dfb4cd89237ab9d2aaa4e7c 2024-11-09T10:18:15,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c348ce33d8ad42f79063ca73cbed8f00 is 50, key is test_row_0/C:col10/1731147494464/Put/seqid=0 2024-11-09T10:18:15,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742403_1579 (size=12301) 2024-11-09T10:18:15,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=80.51 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c348ce33d8ad42f79063ca73cbed8f00 2024-11-09T10:18:15,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/1576af48130742d095e2f7902afb7ac7 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1576af48130742d095e2f7902afb7ac7 2024-11-09T10:18:15,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1576af48130742d095e2f7902afb7ac7, entries=200, sequenceid=532, filesize=39.0 K 2024-11-09T10:18:15,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/20f7b7012dfb4cd89237ab9d2aaa4e7c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/20f7b7012dfb4cd89237ab9d2aaa4e7c 2024-11-09T10:18:15,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/20f7b7012dfb4cd89237ab9d2aaa4e7c, entries=150, sequenceid=532, filesize=12.0 K 2024-11-09T10:18:15,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/c348ce33d8ad42f79063ca73cbed8f00 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c348ce33d8ad42f79063ca73cbed8f00 2024-11-09T10:18:15,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c348ce33d8ad42f79063ca73cbed8f00, entries=150, sequenceid=532, filesize=12.0 K 2024-11-09T10:18:15,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~296.61 KB/303728, currentSize=80.51 KB/82440 for be37ddd6456db463b3a7fa388151b50e in 893ms, sequenceid=532, compaction requested=true 2024-11-09T10:18:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:15,357 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:15,357 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:15,358 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:15,358 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:15,358 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:15,358 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:15,358 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,358 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,358 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6b62f2abe28a41ce950b3802ec6788f3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44450056c21846ca923e0719c9cc4976, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/0aac00805e3d4676bdd9c8664adf9277, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1576af48130742d095e2f7902afb7ac7] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=156.3 K 2024-11-09T10:18:15,358 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/cab9045c37fe4db3b03466f19f25d5d3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/0686517d9003419f98032a9685d2742d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3657361e0b3f4e6a85c840f8385f8a1d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/20f7b7012dfb4cd89237ab9d2aaa4e7c] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=46.0 K 2024-11-09T10:18:15,359 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,359 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6b62f2abe28a41ce950b3802ec6788f3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44450056c21846ca923e0719c9cc4976, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/0aac00805e3d4676bdd9c8664adf9277, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1576af48130742d095e2f7902afb7ac7] 2024-11-09T10:18:15,359 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting cab9045c37fe4db3b03466f19f25d5d3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147490641 2024-11-09T10:18:15,359 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b62f2abe28a41ce950b3802ec6788f3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147490641 2024-11-09T10:18:15,359 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0686517d9003419f98032a9685d2742d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731147491540 2024-11-09T10:18:15,360 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 3657361e0b3f4e6a85c840f8385f8a1d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731147491571 2024-11-09T10:18:15,360 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44450056c21846ca923e0719c9cc4976, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731147491540 2024-11-09T10:18:15,360 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f7b7012dfb4cd89237ab9d2aaa4e7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731147492304 2024-11-09T10:18:15,360 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aac00805e3d4676bdd9c8664adf9277, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731147491571 2024-11-09T10:18:15,360 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1576af48130742d095e2f7902afb7ac7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731147492304 2024-11-09T10:18:15,366 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:15,369 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:15,370 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/2030a1cab58545dca2bb240602e65d82 is 50, key is test_row_0/B:col10/1731147494464/Put/seqid=0 2024-11-09T10:18:15,373 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d6fb38a4ca0a41d4b354f466d963416a_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:15,375 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d6fb38a4ca0a41d4b354f466d963416a_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:15,375 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d6fb38a4ca0a41d4b354f466d963416a_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:15,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742404_1580 (size=12881) 2024-11-09T10:18:15,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742405_1581 (size=4469) 2024-11-09T10:18:15,407 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#484 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:15,407 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a8d8c0840cab4ba7a8f78391d7720c5c is 175, key is test_row_0/A:col10/1731147494464/Put/seqid=0 2024-11-09T10:18:15,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742406_1582 (size=31835) 2024-11-09T10:18:15,415 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a8d8c0840cab4ba7a8f78391d7720c5c as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a8d8c0840cab4ba7a8f78391d7720c5c 2024-11-09T10:18:15,419 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into a8d8c0840cab4ba7a8f78391d7720c5c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:15,419 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:15,420 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=12, startTime=1731147495357; duration=0sec 2024-11-09T10:18:15,420 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:15,420 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:15,420 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:15,421 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:15,421 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:15,421 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,421 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bef15c122333425ab232adecd7c6ba94, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/defb378623754cdc8092439e5cb7f39e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/f542b2461dd1481f9e71df17fb53a98c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c348ce33d8ad42f79063ca73cbed8f00] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=46.0 K 2024-11-09T10:18:15,422 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting bef15c122333425ab232adecd7c6ba94, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731147490641 2024-11-09T10:18:15,422 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting defb378623754cdc8092439e5cb7f39e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731147491540 2024-11-09T10:18:15,422 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting f542b2461dd1481f9e71df17fb53a98c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731147491571 2024-11-09T10:18:15,422 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c348ce33d8ad42f79063ca73cbed8f00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731147492304 2024-11-09T10:18:15,432 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:15,433 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/ad63d5d47e80493da0a3895060a5c270 is 50, key is test_row_0/C:col10/1731147494464/Put/seqid=0 2024-11-09T10:18:15,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742407_1583 (size=12881) 2024-11-09T10:18:15,441 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/ad63d5d47e80493da0a3895060a5c270 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ad63d5d47e80493da0a3895060a5c270 2024-11-09T10:18:15,447 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into ad63d5d47e80493da0a3895060a5c270(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:15,447 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:15,447 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=12, startTime=1731147495357; duration=0sec 2024-11-09T10:18:15,447 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:15,447 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:15,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T10:18:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:15,451 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T10:18:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:15,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fbe3e7723cd1470393388bfbecb3ec30_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147494474/Put/seqid=0 2024-11-09T10:18:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T10:18:15,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742408_1584 (size=12454) 2024-11-09T10:18:15,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:15,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147555651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147555653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147555655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147555657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147555769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147555769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147555769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147555769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,809 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/2030a1cab58545dca2bb240602e65d82 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2030a1cab58545dca2bb240602e65d82 2024-11-09T10:18:15,813 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into 2030a1cab58545dca2bb240602e65d82(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:15,813 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:15,813 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=12, startTime=1731147495357; duration=0sec 2024-11-09T10:18:15,813 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:15,813 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:15,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:15,876 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fbe3e7723cd1470393388bfbecb3ec30_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fbe3e7723cd1470393388bfbecb3ec30_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:15,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/89e0c810e5824f5d9b738540a7e659f4, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:15,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/89e0c810e5824f5d9b738540a7e659f4 is 175, key is test_row_0/A:col10/1731147494474/Put/seqid=0 2024-11-09T10:18:15,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742409_1585 (size=31255) 2024-11-09T10:18:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T10:18:15,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147555979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147555979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147555979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147555979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,280 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=549, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/89e0c810e5824f5d9b738540a7e659f4 2024-11-09T10:18:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/02cf8f2887e04f9ca8f4780651566c26 is 50, key is test_row_0/B:col10/1731147494474/Put/seqid=0 2024-11-09T10:18:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742410_1586 (size=12301) 2024-11-09T10:18:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147556289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147556289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147556289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147556289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147556519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,521 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in be37ddd6456db463b3a7fa388151b50e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:18:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,689 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/02cf8f2887e04f9ca8f4780651566c26 2024-11-09T10:18:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/88f0252bca79468eb9a5c87867db99b6 is 50, key is test_row_0/C:col10/1731147494474/Put/seqid=0 2024-11-09T10:18:16,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742411_1587 (size=12301) 2024-11-09T10:18:16,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/88f0252bca79468eb9a5c87867db99b6 2024-11-09T10:18:16,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/89e0c810e5824f5d9b738540a7e659f4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/89e0c810e5824f5d9b738540a7e659f4 2024-11-09T10:18:16,708 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/89e0c810e5824f5d9b738540a7e659f4, entries=150, sequenceid=549, filesize=30.5 K 2024-11-09T10:18:16,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/02cf8f2887e04f9ca8f4780651566c26 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/02cf8f2887e04f9ca8f4780651566c26 2024-11-09T10:18:16,711 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/02cf8f2887e04f9ca8f4780651566c26, entries=150, sequenceid=549, filesize=12.0 K 2024-11-09T10:18:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/88f0252bca79468eb9a5c87867db99b6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/88f0252bca79468eb9a5c87867db99b6 2024-11-09T10:18:16,714 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/88f0252bca79468eb9a5c87867db99b6, entries=150, sequenceid=549, filesize=12.0 K 2024-11-09T10:18:16,715 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for be37ddd6456db463b3a7fa388151b50e in 1264ms, sequenceid=549, compaction requested=false 2024-11-09T10:18:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-09T10:18:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-09T10:18:16,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-09T10:18:16,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8750 sec 2024-11-09T10:18:16,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8790 sec 2024-11-09T10:18:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:16,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T10:18:16,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:16,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:16,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:16,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:16,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:16,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:16,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109024100c1e07a4471b914f54e13398e0d_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147495650/Put/seqid=0 2024-11-09T10:18:16,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742412_1588 (size=14994) 2024-11-09T10:18:16,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147556815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147556816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147556816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147556817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147556929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147556929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147556929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147556929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T10:18:16,989 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:16,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-09T10:18:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T10:18:16,992 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:16,993 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:16,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T10:18:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147557139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147557139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147557139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147557139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:17,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109024100c1e07a4471b914f54e13398e0d_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109024100c1e07a4471b914f54e13398e0d_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:17,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/acfb577d82a8466cb47fe016a17c010d, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:17,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/acfb577d82a8466cb47fe016a17c010d is 175, key is test_row_0/A:col10/1731147495650/Put/seqid=0 2024-11-09T10:18:17,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742413_1589 (size=39949) 2024-11-09T10:18:17,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T10:18:17,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:17,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:17,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147557449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147557449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147557449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147557449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:17,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:17,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T10:18:17,628 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=568, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/acfb577d82a8466cb47fe016a17c010d 2024-11-09T10:18:17,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/561cc5e173c94cf79e45896cb22dd0bb is 50, key is test_row_0/B:col10/1731147495650/Put/seqid=0 2024-11-09T10:18:17,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742414_1590 (size=12301) 2024-11-09T10:18:17,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/561cc5e173c94cf79e45896cb22dd0bb 2024-11-09T10:18:17,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/5e8349010438414dae8ef0ad037f2e2d is 50, key is test_row_0/C:col10/1731147495650/Put/seqid=0 2024-11-09T10:18:17,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742415_1591 (size=12301) 2024-11-09T10:18:17,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:17,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147557959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147557959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147557959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147557959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:18,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/5e8349010438414dae8ef0ad037f2e2d 2024-11-09T10:18:18,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/acfb577d82a8466cb47fe016a17c010d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/acfb577d82a8466cb47fe016a17c010d 2024-11-09T10:18:18,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/acfb577d82a8466cb47fe016a17c010d, entries=200, sequenceid=568, filesize=39.0 K 2024-11-09T10:18:18,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/561cc5e173c94cf79e45896cb22dd0bb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/561cc5e173c94cf79e45896cb22dd0bb 2024-11-09T10:18:18,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/561cc5e173c94cf79e45896cb22dd0bb, entries=150, sequenceid=568, filesize=12.0 K 2024-11-09T10:18:18,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/5e8349010438414dae8ef0ad037f2e2d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e8349010438414dae8ef0ad037f2e2d 2024-11-09T10:18:18,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:18,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:18,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:18,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:18,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:18,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:18,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e8349010438414dae8ef0ad037f2e2d, entries=150, sequenceid=568, filesize=12.0 K 2024-11-09T10:18:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:18,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for be37ddd6456db463b3a7fa388151b50e in 1263ms, sequenceid=568, compaction requested=true 2024-11-09T10:18:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:18,064 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:18,064 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:18,066 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:18,066 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:18,066 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:18,066 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a8d8c0840cab4ba7a8f78391d7720c5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/89e0c810e5824f5d9b738540a7e659f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/acfb577d82a8466cb47fe016a17c010d] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=100.6 K 2024-11-09T10:18:18,066 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:18,066 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a8d8c0840cab4ba7a8f78391d7720c5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/89e0c810e5824f5d9b738540a7e659f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/acfb577d82a8466cb47fe016a17c010d] 2024-11-09T10:18:18,066 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:18,066 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:18,066 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:18,067 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2030a1cab58545dca2bb240602e65d82, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/02cf8f2887e04f9ca8f4780651566c26, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/561cc5e173c94cf79e45896cb22dd0bb] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=36.6 K 2024-11-09T10:18:18,067 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d8c0840cab4ba7a8f78391d7720c5c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731147492304 2024-11-09T10:18:18,067 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2030a1cab58545dca2bb240602e65d82, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731147492304 2024-11-09T10:18:18,067 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e0c810e5824f5d9b738540a7e659f4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731147494474 2024-11-09T10:18:18,067 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 02cf8f2887e04f9ca8f4780651566c26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731147494474 2024-11-09T10:18:18,067 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting acfb577d82a8466cb47fe016a17c010d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731147495650 2024-11-09T10:18:18,067 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 561cc5e173c94cf79e45896cb22dd0bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731147495650 2024-11-09T10:18:18,072 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:18,075 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:18,075 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/a7891640caec4f6db5438b16dbf6ada8 is 50, key is test_row_0/B:col10/1731147495650/Put/seqid=0 2024-11-09T10:18:18,077 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d16f6833bc364d94b0800fa801241315_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:18,079 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d16f6833bc364d94b0800fa801241315_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:18,080 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d16f6833bc364d94b0800fa801241315_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:18,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742416_1592 (size=12983) 2024-11-09T10:18:18,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742417_1593 (size=4469) 2024-11-09T10:18:18,106 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#493 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:18,107 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/63849691c8d64179bc7778046c5e64f4 is 175, key is test_row_0/A:col10/1731147495650/Put/seqid=0 2024-11-09T10:18:18,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742418_1594 (size=31937) 2024-11-09T10:18:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T10:18:18,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T10:18:18,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:18,215 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T10:18:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:18,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095e7d85732e934a0aa2c52a532f1f8326_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147496806/Put/seqid=0 2024-11-09T10:18:18,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742419_1595 (size=12454) 2024-11-09T10:18:18,497 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/a7891640caec4f6db5438b16dbf6ada8 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a7891640caec4f6db5438b16dbf6ada8 2024-11-09T10:18:18,500 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into a7891640caec4f6db5438b16dbf6ada8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:18,500 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:18,500 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=13, startTime=1731147498064; duration=0sec 2024-11-09T10:18:18,501 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:18,501 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:18,501 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:18,507 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:18,508 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:18,508 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:18,508 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ad63d5d47e80493da0a3895060a5c270, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/88f0252bca79468eb9a5c87867db99b6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e8349010438414dae8ef0ad037f2e2d] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=36.6 K 2024-11-09T10:18:18,508 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting ad63d5d47e80493da0a3895060a5c270, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731147492304 2024-11-09T10:18:18,508 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f0252bca79468eb9a5c87867db99b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731147494474 2024-11-09T10:18:18,509 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8349010438414dae8ef0ad037f2e2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731147495650 2024-11-09T10:18:18,519 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/63849691c8d64179bc7778046c5e64f4 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/63849691c8d64179bc7778046c5e64f4 2024-11-09T10:18:18,522 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:18,523 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/583dfc190f524edfbc96bd9300630533 is 50, key is test_row_0/C:col10/1731147495650/Put/seqid=0 2024-11-09T10:18:18,523 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into 63849691c8d64179bc7778046c5e64f4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:18,523 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:18,523 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=13, startTime=1731147498064; duration=0sec 2024-11-09T10:18:18,523 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:18,523 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:18,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742420_1596 (size=12983) 2024-11-09T10:18:18,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:18,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095e7d85732e934a0aa2c52a532f1f8326_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095e7d85732e934a0aa2c52a532f1f8326_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:18,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/3aa43990bcc54e469d0e3fdcfbb80e4d, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/3aa43990bcc54e469d0e3fdcfbb80e4d is 175, key is test_row_0/A:col10/1731147496806/Put/seqid=0 2024-11-09T10:18:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742421_1597 (size=31255) 2024-11-09T10:18:18,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=582, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/3aa43990bcc54e469d0e3fdcfbb80e4d 2024-11-09T10:18:18,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/acc67e6d15184a23b56c85e76e60de9f is 50, key is test_row_0/B:col10/1731147496806/Put/seqid=0 2024-11-09T10:18:18,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742422_1598 (size=12301) 2024-11-09T10:18:18,681 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/acc67e6d15184a23b56c85e76e60de9f 2024-11-09T10:18:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/8596f37bdb704a29ae58a1328c61433e is 50, key is test_row_0/C:col10/1731147496806/Put/seqid=0 2024-11-09T10:18:18,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742423_1599 (size=12301) 2024-11-09T10:18:18,942 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/583dfc190f524edfbc96bd9300630533 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/583dfc190f524edfbc96bd9300630533 2024-11-09T10:18:18,945 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into 583dfc190f524edfbc96bd9300630533(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:18,945 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:18,945 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=13, startTime=1731147498064; duration=0sec 2024-11-09T10:18:18,945 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:18,945 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:18,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:18,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147558986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147558987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:18,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147558992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147558992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147559099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147559099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147559099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147559100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,118 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/8596f37bdb704a29ae58a1328c61433e 2024-11-09T10:18:19,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/3aa43990bcc54e469d0e3fdcfbb80e4d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/3aa43990bcc54e469d0e3fdcfbb80e4d 2024-11-09T10:18:19,124 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/3aa43990bcc54e469d0e3fdcfbb80e4d, entries=150, sequenceid=582, filesize=30.5 K 2024-11-09T10:18:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/acc67e6d15184a23b56c85e76e60de9f as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/acc67e6d15184a23b56c85e76e60de9f 2024-11-09T10:18:19,127 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/acc67e6d15184a23b56c85e76e60de9f, entries=150, sequenceid=582, filesize=12.0 K 2024-11-09T10:18:19,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/8596f37bdb704a29ae58a1328c61433e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8596f37bdb704a29ae58a1328c61433e 2024-11-09T10:18:19,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8596f37bdb704a29ae58a1328c61433e, entries=150, sequenceid=582, filesize=12.0 K 2024-11-09T10:18:19,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for be37ddd6456db463b3a7fa388151b50e in 915ms, sequenceid=582, compaction requested=false 2024-11-09T10:18:19,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:19,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-09T10:18:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-09T10:18:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T10:18:19,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-09T10:18:19,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1470 sec 2024-11-09T10:18:19,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1540 sec 2024-11-09T10:18:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:19,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T10:18:19,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:19,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:19,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:19,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:19,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:19,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:19,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098794bb230675493b8e82052204c7b317_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147498989/Put/seqid=0 2024-11-09T10:18:19,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742424_1600 (size=14994) 2024-11-09T10:18:19,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147559321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147559321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147559322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147559326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147559428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147559428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147559428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147559439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147559638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147559639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147559639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147559649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,720 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:19,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098794bb230675493b8e82052204c7b317_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098794bb230675493b8e82052204c7b317_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:19,724 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/55c8e9cce55442b9934530ac83cc98fd, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:19,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/55c8e9cce55442b9934530ac83cc98fd is 175, key is test_row_0/A:col10/1731147498989/Put/seqid=0 2024-11-09T10:18:19,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742425_1601 (size=39949) 2024-11-09T10:18:19,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=605, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/55c8e9cce55442b9934530ac83cc98fd 2024-11-09T10:18:19,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/2362dcbf6c504ca292a0b7bb97dc4980 is 50, key is test_row_0/B:col10/1731147498989/Put/seqid=0 2024-11-09T10:18:19,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742426_1602 (size=12301) 2024-11-09T10:18:19,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/2362dcbf6c504ca292a0b7bb97dc4980 2024-11-09T10:18:19,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/0a7f029adf064cfd9d0dc90d3f4a55bd is 50, key is test_row_0/C:col10/1731147498989/Put/seqid=0 2024-11-09T10:18:19,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742427_1603 (size=12301) 2024-11-09T10:18:19,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147559948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147559948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147559948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:19,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147559959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/0a7f029adf064cfd9d0dc90d3f4a55bd 2024-11-09T10:18:20,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/55c8e9cce55442b9934530ac83cc98fd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/55c8e9cce55442b9934530ac83cc98fd 2024-11-09T10:18:20,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/55c8e9cce55442b9934530ac83cc98fd, entries=200, sequenceid=605, filesize=39.0 K 2024-11-09T10:18:20,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/2362dcbf6c504ca292a0b7bb97dc4980 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2362dcbf6c504ca292a0b7bb97dc4980 2024-11-09T10:18:20,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2362dcbf6c504ca292a0b7bb97dc4980, entries=150, sequenceid=605, filesize=12.0 K 2024-11-09T10:18:20,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/0a7f029adf064cfd9d0dc90d3f4a55bd as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/0a7f029adf064cfd9d0dc90d3f4a55bd 2024-11-09T10:18:20,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/0a7f029adf064cfd9d0dc90d3f4a55bd, entries=150, sequenceid=605, filesize=12.0 K 2024-11-09T10:18:20,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for be37ddd6456db463b3a7fa388151b50e in 851ms, sequenceid=605, compaction requested=true 2024-11-09T10:18:20,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:20,162 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:20,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:20,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:20,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:20,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:20,163 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:20,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:20,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:20,163 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:20,163 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:20,163 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:20,163 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/63849691c8d64179bc7778046c5e64f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/3aa43990bcc54e469d0e3fdcfbb80e4d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/55c8e9cce55442b9934530ac83cc98fd] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=100.7 K 2024-11-09T10:18:20,163 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:20,163 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/63849691c8d64179bc7778046c5e64f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/3aa43990bcc54e469d0e3fdcfbb80e4d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/55c8e9cce55442b9934530ac83cc98fd] 2024-11-09T10:18:20,163 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:20,163 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:20,163 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:20,164 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a7891640caec4f6db5438b16dbf6ada8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/acc67e6d15184a23b56c85e76e60de9f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2362dcbf6c504ca292a0b7bb97dc4980] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=36.7 K 2024-11-09T10:18:20,164 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63849691c8d64179bc7778046c5e64f4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731147495650 2024-11-09T10:18:20,164 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa43990bcc54e469d0e3fdcfbb80e4d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=582, earliestPutTs=1731147496806 2024-11-09T10:18:20,164 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting a7891640caec4f6db5438b16dbf6ada8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731147495650 2024-11-09T10:18:20,164 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c8e9cce55442b9934530ac83cc98fd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731147498986 2024-11-09T10:18:20,164 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting acc67e6d15184a23b56c85e76e60de9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=582, earliestPutTs=1731147496806 2024-11-09T10:18:20,164 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2362dcbf6c504ca292a0b7bb97dc4980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731147498989 2024-11-09T10:18:20,169 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:20,170 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:20,170 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109f6d32ef6cf3f4630b350a01e980a795c_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:20,170 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/2fadc15ee00e4e2088e8964bf0fefd68 is 50, key is test_row_0/B:col10/1731147498989/Put/seqid=0 2024-11-09T10:18:20,173 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109f6d32ef6cf3f4630b350a01e980a795c_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:20,173 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f6d32ef6cf3f4630b350a01e980a795c_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:20,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742428_1604 (size=13085) 2024-11-09T10:18:20,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742429_1605 (size=4469) 2024-11-09T10:18:20,190 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/2fadc15ee00e4e2088e8964bf0fefd68 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2fadc15ee00e4e2088e8964bf0fefd68 2024-11-09T10:18:20,193 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into 2fadc15ee00e4e2088e8964bf0fefd68(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:20,193 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:20,193 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=13, startTime=1731147500163; duration=0sec 2024-11-09T10:18:20,194 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:20,194 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:20,194 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T10:18:20,195 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T10:18:20,195 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:20,195 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:20,195 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/583dfc190f524edfbc96bd9300630533, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8596f37bdb704a29ae58a1328c61433e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/0a7f029adf064cfd9d0dc90d3f4a55bd] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=36.7 K 2024-11-09T10:18:20,195 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 583dfc190f524edfbc96bd9300630533, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731147495650 2024-11-09T10:18:20,195 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 8596f37bdb704a29ae58a1328c61433e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=582, earliestPutTs=1731147496806 2024-11-09T10:18:20,196 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7f029adf064cfd9d0dc90d3f4a55bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731147498989 2024-11-09T10:18:20,202 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:20,202 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/15f566f34cc94553aa4fc089cc39cf5d is 50, key is test_row_0/C:col10/1731147498989/Put/seqid=0 2024-11-09T10:18:20,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742430_1606 (size=13085) 2024-11-09T10:18:20,213 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/15f566f34cc94553aa4fc089cc39cf5d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/15f566f34cc94553aa4fc089cc39cf5d 2024-11-09T10:18:20,216 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into 15f566f34cc94553aa4fc089cc39cf5d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:20,216 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:20,216 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=13, startTime=1731147500163; duration=0sec 2024-11-09T10:18:20,216 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:20,216 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:20,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T10:18:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:20,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109097e498f11354d9cae26e4daaa2ce3e8_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147500460/Put/seqid=0 2024-11-09T10:18:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742431_1607 (size=12454) 2024-11-09T10:18:20,470 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:20,474 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109097e498f11354d9cae26e4daaa2ce3e8_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109097e498f11354d9cae26e4daaa2ce3e8_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:20,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a1390030ac8949ce98971d4efd468782, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:20,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a1390030ac8949ce98971d4efd468782 is 175, key is test_row_0/A:col10/1731147500460/Put/seqid=0 2024-11-09T10:18:20,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742432_1608 (size=31255) 2024-11-09T10:18:20,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147560524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147560531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147560534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147560533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43804 deadline: 1731147560558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,562 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e067b2c2c7b3,38427,1731147328756 for 'test_row_2' in be37ddd6456db463b3a7fa388151b50e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T10:18:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,587 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#503 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:20,588 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c49f1d5cdaf34d75b3355c2e3f427efa is 175, key is test_row_0/A:col10/1731147498989/Put/seqid=0 2024-11-09T10:18:20,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742433_1609 (size=32039) 2024-11-09T10:18:20,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147560639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147560648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147560649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147560649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147560849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147560859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147560859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147560859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:20,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=620, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a1390030ac8949ce98971d4efd468782 2024-11-09T10:18:20,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/67bee43d8ce24fb3a671fcd1bc7a5006 is 50, key is test_row_0/B:col10/1731147500460/Put/seqid=0 2024-11-09T10:18:20,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742434_1610 (size=12301) 2024-11-09T10:18:21,001 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/c49f1d5cdaf34d75b3355c2e3f427efa as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c49f1d5cdaf34d75b3355c2e3f427efa 2024-11-09T10:18:21,004 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into c49f1d5cdaf34d75b3355c2e3f427efa(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:21,004 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:21,004 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=13, startTime=1731147500162; duration=0sec 2024-11-09T10:18:21,004 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:21,004 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T10:18:21,149 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:21,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-09T10:18:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T10:18:21,153 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:21,154 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:21,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:21,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147561159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147561168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147561168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147561168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T10:18:21,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/67bee43d8ce24fb3a671fcd1bc7a5006 2024-11-09T10:18:21,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/9a1dca3e57fb4135bf4e789d8731d914 is 50, key is test_row_0/C:col10/1731147500460/Put/seqid=0 2024-11-09T10:18:21,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742435_1611 (size=12301) 2024-11-09T10:18:21,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/9a1dca3e57fb4135bf4e789d8731d914 2024-11-09T10:18:21,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T10:18:21,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:21,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:21,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:21,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:21,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:21,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/a1390030ac8949ce98971d4efd468782 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a1390030ac8949ce98971d4efd468782 2024-11-09T10:18:21,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a1390030ac8949ce98971d4efd468782, entries=150, sequenceid=620, filesize=30.5 K 2024-11-09T10:18:21,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/67bee43d8ce24fb3a671fcd1bc7a5006 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/67bee43d8ce24fb3a671fcd1bc7a5006 2024-11-09T10:18:21,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/67bee43d8ce24fb3a671fcd1bc7a5006, entries=150, sequenceid=620, filesize=12.0 K 2024-11-09T10:18:21,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/9a1dca3e57fb4135bf4e789d8731d914 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/9a1dca3e57fb4135bf4e789d8731d914 2024-11-09T10:18:21,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/9a1dca3e57fb4135bf4e789d8731d914, entries=150, sequenceid=620, filesize=12.0 K 2024-11-09T10:18:21,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for be37ddd6456db463b3a7fa388151b50e in 868ms, sequenceid=620, compaction requested=false 2024-11-09T10:18:21,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:21,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T10:18:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:21,462 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T10:18:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110943cd2ee14b254323be6446c5b47683cb_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147500532/Put/seqid=0 2024-11-09T10:18:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T10:18:21,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742436_1612 (size=12454) 2024-11-09T10:18:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:21,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147561683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147561686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147561687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147561688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T10:18:21,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147561799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147561799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147561799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147561799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:21,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:21,874 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110943cd2ee14b254323be6446c5b47683cb_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110943cd2ee14b254323be6446c5b47683cb_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:21,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8a57cac85c014b63a94eaee6d2b8fa88, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:21,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8a57cac85c014b63a94eaee6d2b8fa88 is 175, key is test_row_0/A:col10/1731147500532/Put/seqid=0 2024-11-09T10:18:21,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742437_1613 (size=31255) 2024-11-09T10:18:21,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=640, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8a57cac85c014b63a94eaee6d2b8fa88 2024-11-09T10:18:21,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/3bd5aca5e852490683107c70ce3eac8b is 50, key is test_row_0/B:col10/1731147500532/Put/seqid=0 2024-11-09T10:18:21,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742438_1614 (size=12301) 2024-11-09T10:18:21,886 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/3bd5aca5e852490683107c70ce3eac8b 2024-11-09T10:18:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/fba0ddd8246f444ba2fa492a69a55a91 is 50, key is test_row_0/C:col10/1731147500532/Put/seqid=0 2024-11-09T10:18:21,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742439_1615 (size=12301) 2024-11-09T10:18:22,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147562009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147562009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147562009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147562009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T10:18:22,310 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/fba0ddd8246f444ba2fa492a69a55a91 2024-11-09T10:18:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8a57cac85c014b63a94eaee6d2b8fa88 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8a57cac85c014b63a94eaee6d2b8fa88 2024-11-09T10:18:22,316 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8a57cac85c014b63a94eaee6d2b8fa88, entries=150, sequenceid=640, filesize=30.5 K 2024-11-09T10:18:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/3bd5aca5e852490683107c70ce3eac8b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3bd5aca5e852490683107c70ce3eac8b 2024-11-09T10:18:22,321 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3bd5aca5e852490683107c70ce3eac8b, entries=150, sequenceid=640, filesize=12.0 K 2024-11-09T10:18:22,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/fba0ddd8246f444ba2fa492a69a55a91 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/fba0ddd8246f444ba2fa492a69a55a91 2024-11-09T10:18:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147562319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147562319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147562319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147562319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,325 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/fba0ddd8246f444ba2fa492a69a55a91, entries=150, sequenceid=640, filesize=12.0 K 2024-11-09T10:18:22,326 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for be37ddd6456db463b3a7fa388151b50e in 864ms, sequenceid=640, compaction requested=true 2024-11-09T10:18:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-09T10:18:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-09T10:18:22,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-09T10:18:22,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1730 sec 2024-11-09T10:18:22,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.1800 sec 2024-11-09T10:18:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:22,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T10:18:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:22,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110983b0dfaa59094ca0bca812f1c436037d_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147502832/Put/seqid=0 2024-11-09T10:18:22,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742440_1616 (size=17534) 2024-11-09T10:18:22,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147562852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147562853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147562854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147562855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147562959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147562968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147562969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147562969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147563169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147563178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147563178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147563178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:23,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110983b0dfaa59094ca0bca812f1c436037d_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110983b0dfaa59094ca0bca812f1c436037d_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:23,245 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/d52cf59c914a410084eb1aaba61eabc2, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:23,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/d52cf59c914a410084eb1aaba61eabc2 is 175, key is test_row_0/A:col10/1731147502832/Put/seqid=0 2024-11-09T10:18:23,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742441_1617 (size=48639) 2024-11-09T10:18:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T10:18:23,299 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:23,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T10:18:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-09T10:18:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T10:18:23,300 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T10:18:23,301 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T10:18:23,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T10:18:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T10:18:23,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147563479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147563489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147563489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147563489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T10:18:23,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=655, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/d52cf59c914a410084eb1aaba61eabc2 2024-11-09T10:18:23,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/58e145d8dc8c4768b01b7f36f1962103 is 50, key is test_row_0/B:col10/1731147502832/Put/seqid=0 2024-11-09T10:18:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742442_1618 (size=12301) 2024-11-09T10:18:23,732 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-11-09T10:18:23,732 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:23,732 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,732 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:23,732 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-09T10:18:23,732 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:23,732 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,732 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:23,733 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-09T10:18:23,733 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:23,733 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,733 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:23,736 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-09T10:18:23,736 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:23,736 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,736 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:23,736 INFO [Thread-2310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2310. 2024-11-09T10:18:23,736 DEBUG [Thread-2310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:23,736 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,737 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:23,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:23,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:23,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:23,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:23,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T10:18:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43820 deadline: 1731147563989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43784 deadline: 1731147563998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T10:18:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731147563998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731147563998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 , the old value is region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., hostname=e067b2c2c7b3,38427,1731147328756, seqNum=5 because the exception is null or not the one we care about 2024-11-09T10:18:24,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/58e145d8dc8c4768b01b7f36f1962103 2024-11-09T10:18:24,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/ba0cb060bd1b494ea049da6145e993ad is 50, key is test_row_0/C:col10/1731147502832/Put/seqid=0 2024-11-09T10:18:24,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742443_1619 (size=12301) 2024-11-09T10:18:24,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:24,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:24,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T10:18:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T10:18:24,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/ba0cb060bd1b494ea049da6145e993ad 2024-11-09T10:18:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/d52cf59c914a410084eb1aaba61eabc2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/d52cf59c914a410084eb1aaba61eabc2 2024-11-09T10:18:24,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/d52cf59c914a410084eb1aaba61eabc2, entries=250, sequenceid=655, filesize=47.5 K 2024-11-09T10:18:24,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/58e145d8dc8c4768b01b7f36f1962103 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/58e145d8dc8c4768b01b7f36f1962103 2024-11-09T10:18:24,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/58e145d8dc8c4768b01b7f36f1962103, entries=150, sequenceid=655, filesize=12.0 K 2024-11-09T10:18:24,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/ba0cb060bd1b494ea049da6145e993ad as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ba0cb060bd1b494ea049da6145e993ad 2024-11-09T10:18:24,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ba0cb060bd1b494ea049da6145e993ad, entries=150, sequenceid=655, filesize=12.0 K 2024-11-09T10:18:24,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for be37ddd6456db463b3a7fa388151b50e in 1649ms, sequenceid=655, compaction requested=true 2024-11-09T10:18:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T10:18:24,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:24,482 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:24,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T10:18:24,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:24,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be37ddd6456db463b3a7fa388151b50e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T10:18:24,482 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:24,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:24,482 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:24,482 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:24,482 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/B is initiating minor compaction (all files) 2024-11-09T10:18:24,482 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/A is initiating minor compaction (all files) 2024-11-09T10:18:24,482 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/A in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,482 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/B in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,482 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c49f1d5cdaf34d75b3355c2e3f427efa, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a1390030ac8949ce98971d4efd468782, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8a57cac85c014b63a94eaee6d2b8fa88, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/d52cf59c914a410084eb1aaba61eabc2] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=139.8 K 2024-11-09T10:18:24,482 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2fadc15ee00e4e2088e8964bf0fefd68, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/67bee43d8ce24fb3a671fcd1bc7a5006, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3bd5aca5e852490683107c70ce3eac8b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/58e145d8dc8c4768b01b7f36f1962103] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=48.8 K 2024-11-09T10:18:24,482 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,482 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. files: [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c49f1d5cdaf34d75b3355c2e3f427efa, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a1390030ac8949ce98971d4efd468782, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8a57cac85c014b63a94eaee6d2b8fa88, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/d52cf59c914a410084eb1aaba61eabc2] 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fadc15ee00e4e2088e8964bf0fefd68, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731147498989 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49f1d5cdaf34d75b3355c2e3f427efa, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731147498989 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bee43d8ce24fb3a671fcd1bc7a5006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1731147499317 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1390030ac8949ce98971d4efd468782, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1731147499317 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd5aca5e852490683107c70ce3eac8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1731147500522 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a57cac85c014b63a94eaee6d2b8fa88, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1731147500522 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e145d8dc8c4768b01b7f36f1962103, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1731147501687 2024-11-09T10:18:24,483 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] compactions.Compactor(225): Compacting d52cf59c914a410084eb1aaba61eabc2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1731147501682 2024-11-09T10:18:24,488 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:24,489 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411097b87c225754241bfbd8a2f2d40f8df2e_be37ddd6456db463b3a7fa388151b50e store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:24,489 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:24,490 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/d4fd21f66b444b8182f21de17287f9f2 is 50, key is test_row_0/B:col10/1731147502832/Put/seqid=0 2024-11-09T10:18:24,494 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411097b87c225754241bfbd8a2f2d40f8df2e_be37ddd6456db463b3a7fa388151b50e, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:24,494 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097b87c225754241bfbd8a2f2d40f8df2e_be37ddd6456db463b3a7fa388151b50e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:24,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742444_1620 (size=13221) 2024-11-09T10:18:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742445_1621 (size=4469) 2024-11-09T10:18:24,497 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#A#compaction#515 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:24,498 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/889856d3842f46cca33cbfe07d63ff98 is 175, key is test_row_0/A:col10/1731147502832/Put/seqid=0 2024-11-09T10:18:24,500 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/d4fd21f66b444b8182f21de17287f9f2 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/d4fd21f66b444b8182f21de17287f9f2 2024-11-09T10:18:24,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742446_1622 (size=32175) 2024-11-09T10:18:24,503 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/B of be37ddd6456db463b3a7fa388151b50e into d4fd21f66b444b8182f21de17287f9f2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:24,503 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:24,504 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/B, priority=12, startTime=1731147504482; duration=0sec 2024-11-09T10:18:24,504 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T10:18:24,504 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:B 2024-11-09T10:18:24,504 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T10:18:24,505 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T10:18:24,505 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1541): be37ddd6456db463b3a7fa388151b50e/C is initiating minor compaction (all files) 2024-11-09T10:18:24,506 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be37ddd6456db463b3a7fa388151b50e/C in TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,506 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/15f566f34cc94553aa4fc089cc39cf5d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/9a1dca3e57fb4135bf4e789d8731d914, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/fba0ddd8246f444ba2fa492a69a55a91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ba0cb060bd1b494ea049da6145e993ad] into tmpdir=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp, totalSize=48.8 K 2024-11-09T10:18:24,506 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 15f566f34cc94553aa4fc089cc39cf5d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731147498989 2024-11-09T10:18:24,506 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a1dca3e57fb4135bf4e789d8731d914, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1731147499317 2024-11-09T10:18:24,506 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting fba0ddd8246f444ba2fa492a69a55a91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1731147500522 2024-11-09T10:18:24,506 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] compactions.Compactor(225): Compacting ba0cb060bd1b494ea049da6145e993ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1731147501687 2024-11-09T10:18:24,517 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be37ddd6456db463b3a7fa388151b50e#C#compaction#516 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-09T10:18:24,518 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/6817071ef7794cb3b19444bc35e30c02 is 50, key is test_row_0/C:col10/1731147502832/Put/seqid=0 2024-11-09T10:18:24,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38427 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T10:18:24,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:24,520 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T10:18:24,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:24,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:24,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:24,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:24,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:24,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:24,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742447_1623 (size=13221) 2024-11-09T10:18:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110953aecfa6aa7e490e9068873b366e158c_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_0/A:col10/1731147502852/Put/seqid=0 2024-11-09T10:18:24,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742448_1624 (size=12454) 2024-11-09T10:18:24,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:24,538 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110953aecfa6aa7e490e9068873b366e158c_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110953aecfa6aa7e490e9068873b366e158c_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:24,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/e85dbdc846094a10bba78b0f01226b3e, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:24,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/e85dbdc846094a10bba78b0f01226b3e is 175, key is test_row_0/A:col10/1731147502852/Put/seqid=0 2024-11-09T10:18:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742449_1625 (size=31255) 2024-11-09T10:18:24,905 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/889856d3842f46cca33cbfe07d63ff98 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/889856d3842f46cca33cbfe07d63ff98 2024-11-09T10:18:24,908 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/A of be37ddd6456db463b3a7fa388151b50e into 889856d3842f46cca33cbfe07d63ff98(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:24,908 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:24,908 INFO [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/A, priority=12, startTime=1731147504481; duration=0sec 2024-11-09T10:18:24,908 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:24,908 DEBUG [RS:0;e067b2c2c7b3:38427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:A 2024-11-09T10:18:24,925 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/6817071ef7794cb3b19444bc35e30c02 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/6817071ef7794cb3b19444bc35e30c02 2024-11-09T10:18:24,929 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be37ddd6456db463b3a7fa388151b50e/C of be37ddd6456db463b3a7fa388151b50e into 6817071ef7794cb3b19444bc35e30c02(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T10:18:24,929 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:24,929 INFO [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., storeName=be37ddd6456db463b3a7fa388151b50e/C, priority=12, startTime=1731147504482; duration=0sec 2024-11-09T10:18:24,929 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T10:18:24,929 DEBUG [RS:0;e067b2c2c7b3:38427-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be37ddd6456db463b3a7fa388151b50e:C 2024-11-09T10:18:24,944 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=673, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/e85dbdc846094a10bba78b0f01226b3e 2024-11-09T10:18:24,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/9c0f6c95614d4e528123dce74cdccbdb is 50, key is test_row_0/B:col10/1731147502852/Put/seqid=0 2024-11-09T10:18:24,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742450_1626 (size=12301) 2024-11-09T10:18:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38427 {}] regionserver.HRegion(8855): Flush requested on be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:25,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. as already flushing 2024-11-09T10:18:25,000 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-11-09T10:18:25,000 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:25,000 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,000 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:25,011 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-11-09T10:18:25,011 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:25,011 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,011 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-11-09T10:18:25,011 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-11-09T10:18:25,011 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:25,011 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,011 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:25,011 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,011 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:25,011 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,011 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:25,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:25,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:25,352 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/9c0f6c95614d4e528123dce74cdccbdb 2024-11-09T10:18:25,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/5e50da3947614ee88aed783fd56a9b0b is 50, key is test_row_0/C:col10/1731147502852/Put/seqid=0 2024-11-09T10:18:25,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742451_1627 (size=12301) 2024-11-09T10:18:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T10:18:25,761 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/5e50da3947614ee88aed783fd56a9b0b 2024-11-09T10:18:25,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/e85dbdc846094a10bba78b0f01226b3e as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/e85dbdc846094a10bba78b0f01226b3e 2024-11-09T10:18:25,767 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/e85dbdc846094a10bba78b0f01226b3e, entries=150, sequenceid=673, filesize=30.5 K 2024-11-09T10:18:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/9c0f6c95614d4e528123dce74cdccbdb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9c0f6c95614d4e528123dce74cdccbdb 2024-11-09T10:18:25,770 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9c0f6c95614d4e528123dce74cdccbdb, entries=150, sequenceid=673, filesize=12.0 K 2024-11-09T10:18:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/5e50da3947614ee88aed783fd56a9b0b as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e50da3947614ee88aed783fd56a9b0b 2024-11-09T10:18:25,772 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e50da3947614ee88aed783fd56a9b0b, entries=150, sequenceid=673, filesize=12.0 K 2024-11-09T10:18:25,773 INFO [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for be37ddd6456db463b3a7fa388151b50e in 1253ms, sequenceid=673, compaction requested=false 2024-11-09T10:18:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for be37ddd6456db463b3a7fa388151b50e: 2024-11-09T10:18:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e067b2c2c7b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-09T10:18:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-09T10:18:25,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-09T10:18:25,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4730 sec 2024-11-09T10:18:25,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.4770 sec 2024-11-09T10:18:26,957 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T10:18:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T10:18:27,479 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:30,640 INFO [Thread-2299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2299. 2024-11-09T10:18:30,640 DEBUG [Thread-2299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T10:18:30,640 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:30,640 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:30,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2773 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2792 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2771 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2772 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2800 2024-11-09T10:18:30,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T10:18:30,640 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:18:30,640 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:18:30,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:30,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:30,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:30,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T10:18:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T10:18:30,643 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147510643"}]},"ts":"1731147510643"} 2024-11-09T10:18:30,644 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T10:18:30,644 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T10:18:30,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T10:18:30,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, UNASSIGN}] 2024-11-09T10:18:30,646 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, UNASSIGN 2024-11-09T10:18:30,647 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=CLOSING, regionLocation=e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:30,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, UNASSIGN because future has completed 2024-11-09T10:18:30,648 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T10:18:30,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756}] 2024-11-09T10:18:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T10:18:30,799 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:30,799 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing be37ddd6456db463b3a7fa388151b50e, disabling compactions & flushes 2024-11-09T10:18:30,800 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. after waiting 0 ms 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:30,800 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing be37ddd6456db463b3a7fa388151b50e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=A 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=B 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be37ddd6456db463b3a7fa388151b50e, store=C 2024-11-09T10:18:30,800 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T10:18:30,808 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097b64e148d5914eb8943d62c63cd55e38_be37ddd6456db463b3a7fa388151b50e is 50, key is test_row_1/A:col10/1731147504999/Put/seqid=0 2024-11-09T10:18:30,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742452_1628 (size=9914) 2024-11-09T10:18:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T10:18:31,211 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T10:18:31,215 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097b64e148d5914eb8943d62c63cd55e38_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b64e148d5914eb8943d62c63cd55e38_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:31,215 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8e7ef11bbbf64de681719fb8a3147ff5, store: [table=TestAcidGuarantees family=A region=be37ddd6456db463b3a7fa388151b50e] 2024-11-09T10:18:31,216 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8e7ef11bbbf64de681719fb8a3147ff5 is 175, key is test_row_1/A:col10/1731147504999/Put/seqid=0 2024-11-09T10:18:31,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742453_1629 (size=22561) 2024-11-09T10:18:31,238 DEBUG [regionserver/e067b2c2c7b3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-09T10:18:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T10:18:31,619 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=683, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8e7ef11bbbf64de681719fb8a3147ff5 2024-11-09T10:18:31,624 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/4ae8724bfe4b4b579a94594d9f9f304d is 50, key is test_row_1/B:col10/1731147504999/Put/seqid=0 2024-11-09T10:18:31,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742454_1630 (size=9857) 2024-11-09T10:18:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T10:18:32,030 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/4ae8724bfe4b4b579a94594d9f9f304d 2024-11-09T10:18:32,034 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/3e3631205ef74503a5e5a67c12eebd56 is 50, key is test_row_1/C:col10/1731147504999/Put/seqid=0 2024-11-09T10:18:32,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742455_1631 (size=9857) 2024-11-09T10:18:32,438 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/3e3631205ef74503a5e5a67c12eebd56 2024-11-09T10:18:32,441 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/A/8e7ef11bbbf64de681719fb8a3147ff5 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8e7ef11bbbf64de681719fb8a3147ff5 2024-11-09T10:18:32,444 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8e7ef11bbbf64de681719fb8a3147ff5, entries=100, sequenceid=683, filesize=22.0 K 2024-11-09T10:18:32,444 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/B/4ae8724bfe4b4b579a94594d9f9f304d as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/4ae8724bfe4b4b579a94594d9f9f304d 2024-11-09T10:18:32,447 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/4ae8724bfe4b4b579a94594d9f9f304d, entries=100, sequenceid=683, filesize=9.6 K 2024-11-09T10:18:32,447 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/.tmp/C/3e3631205ef74503a5e5a67c12eebd56 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/3e3631205ef74503a5e5a67c12eebd56 2024-11-09T10:18:32,450 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/3e3631205ef74503a5e5a67c12eebd56, entries=100, sequenceid=683, filesize=9.6 K 2024-11-09T10:18:32,450 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for be37ddd6456db463b3a7fa388151b50e in 1650ms, sequenceid=683, compaction requested=true 2024-11-09T10:18:32,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/82f6bc09d5d24d969aa4cefcef429a9a, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/b55a19b73da043298ce030e52f68f648, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/2b78d77a643643a79273c6e4618db847, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44412fb044a74ec5836b7e4ca1991f1d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c3f7bf92143440a694bb6f95c3bd8e2d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a80a780ed0f84adc941272a6cad8690c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1760682999604cb282382fb316653d56, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c4078915945b48af85377d44abb1c213, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/49d8e21353b34c9993dd3a6a8a949572, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6945a7e2f9e6429288a5f2f019ef31c2, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/f1e0827456154986afc8898f6c6ec929, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/48e828953f0c495b8df5880eef3927e6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/4a6f9cdd48cf4d04a39632647c9345b1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6b62f2abe28a41ce950b3802ec6788f3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44450056c21846ca923e0719c9cc4976, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/0aac00805e3d4676bdd9c8664adf9277, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1576af48130742d095e2f7902afb7ac7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a8d8c0840cab4ba7a8f78391d7720c5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/89e0c810e5824f5d9b738540a7e659f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/acfb577d82a8466cb47fe016a17c010d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/63849691c8d64179bc7778046c5e64f4, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/3aa43990bcc54e469d0e3fdcfbb80e4d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/55c8e9cce55442b9934530ac83cc98fd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c49f1d5cdaf34d75b3355c2e3f427efa, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a1390030ac8949ce98971d4efd468782, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8a57cac85c014b63a94eaee6d2b8fa88, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/d52cf59c914a410084eb1aaba61eabc2] to archive 2024-11-09T10:18:32,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:18:32,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/82f6bc09d5d24d969aa4cefcef429a9a to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/82f6bc09d5d24d969aa4cefcef429a9a 2024-11-09T10:18:32,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/b55a19b73da043298ce030e52f68f648 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/b55a19b73da043298ce030e52f68f648 2024-11-09T10:18:32,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/2b78d77a643643a79273c6e4618db847 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/2b78d77a643643a79273c6e4618db847 2024-11-09T10:18:32,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44412fb044a74ec5836b7e4ca1991f1d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44412fb044a74ec5836b7e4ca1991f1d 2024-11-09T10:18:32,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c3f7bf92143440a694bb6f95c3bd8e2d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c3f7bf92143440a694bb6f95c3bd8e2d 2024-11-09T10:18:32,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a80a780ed0f84adc941272a6cad8690c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a80a780ed0f84adc941272a6cad8690c 2024-11-09T10:18:32,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1760682999604cb282382fb316653d56 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1760682999604cb282382fb316653d56 2024-11-09T10:18:32,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c4078915945b48af85377d44abb1c213 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c4078915945b48af85377d44abb1c213 2024-11-09T10:18:32,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/49d8e21353b34c9993dd3a6a8a949572 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/49d8e21353b34c9993dd3a6a8a949572 2024-11-09T10:18:32,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6945a7e2f9e6429288a5f2f019ef31c2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6945a7e2f9e6429288a5f2f019ef31c2 2024-11-09T10:18:32,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/f1e0827456154986afc8898f6c6ec929 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/f1e0827456154986afc8898f6c6ec929 2024-11-09T10:18:32,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/48e828953f0c495b8df5880eef3927e6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/48e828953f0c495b8df5880eef3927e6 2024-11-09T10:18:32,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/4a6f9cdd48cf4d04a39632647c9345b1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/4a6f9cdd48cf4d04a39632647c9345b1 2024-11-09T10:18:32,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6b62f2abe28a41ce950b3802ec6788f3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/6b62f2abe28a41ce950b3802ec6788f3 2024-11-09T10:18:32,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44450056c21846ca923e0719c9cc4976 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/44450056c21846ca923e0719c9cc4976 2024-11-09T10:18:32,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/0aac00805e3d4676bdd9c8664adf9277 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/0aac00805e3d4676bdd9c8664adf9277 2024-11-09T10:18:32,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1576af48130742d095e2f7902afb7ac7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/1576af48130742d095e2f7902afb7ac7 2024-11-09T10:18:32,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a8d8c0840cab4ba7a8f78391d7720c5c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a8d8c0840cab4ba7a8f78391d7720c5c 2024-11-09T10:18:32,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/89e0c810e5824f5d9b738540a7e659f4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/89e0c810e5824f5d9b738540a7e659f4 2024-11-09T10:18:32,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/acfb577d82a8466cb47fe016a17c010d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/acfb577d82a8466cb47fe016a17c010d 2024-11-09T10:18:32,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/63849691c8d64179bc7778046c5e64f4 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/63849691c8d64179bc7778046c5e64f4 2024-11-09T10:18:32,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/3aa43990bcc54e469d0e3fdcfbb80e4d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/3aa43990bcc54e469d0e3fdcfbb80e4d 2024-11-09T10:18:32,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/55c8e9cce55442b9934530ac83cc98fd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/55c8e9cce55442b9934530ac83cc98fd 2024-11-09T10:18:32,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c49f1d5cdaf34d75b3355c2e3f427efa to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/c49f1d5cdaf34d75b3355c2e3f427efa 2024-11-09T10:18:32,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a1390030ac8949ce98971d4efd468782 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/a1390030ac8949ce98971d4efd468782 2024-11-09T10:18:32,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8a57cac85c014b63a94eaee6d2b8fa88 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8a57cac85c014b63a94eaee6d2b8fa88 2024-11-09T10:18:32,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/d52cf59c914a410084eb1aaba61eabc2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/d52cf59c914a410084eb1aaba61eabc2 2024-11-09T10:18:32,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/96c5ae85acf24480b8e8dee445a3ad09, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9ab93d6e22254c899664426a4f169010, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/78fd1dd6f2314e468d8e8aef172ed50c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/6cd2a2a6bfe849eeaa23d1f712030e67, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/347f7aca24324496a3aa921de6b4832c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/dfde14536f15484398485bd2ec8296a6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/b2c82d3b7c0a4bb68d6636ea2708e8c5, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/5e82549f4cdc4521ab7c44130aa58358, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a8e2d2be802b4658b19c5deef8021a91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/af80eda4b8b94e648b39c4b19e514545, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/7b34129a600d4b48b3b632f1140fd7c7, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e06468637b0e47d0bdd27844262a7fec, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/cab9045c37fe4db3b03466f19f25d5d3, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e4410e5d6bc04c39945af4defa557eca, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/0686517d9003419f98032a9685d2742d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3657361e0b3f4e6a85c840f8385f8a1d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2030a1cab58545dca2bb240602e65d82, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/20f7b7012dfb4cd89237ab9d2aaa4e7c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/02cf8f2887e04f9ca8f4780651566c26, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a7891640caec4f6db5438b16dbf6ada8, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/561cc5e173c94cf79e45896cb22dd0bb, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/acc67e6d15184a23b56c85e76e60de9f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2fadc15ee00e4e2088e8964bf0fefd68, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2362dcbf6c504ca292a0b7bb97dc4980, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/67bee43d8ce24fb3a671fcd1bc7a5006, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3bd5aca5e852490683107c70ce3eac8b, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/58e145d8dc8c4768b01b7f36f1962103] to archive 2024-11-09T10:18:32,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:18:32,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/96c5ae85acf24480b8e8dee445a3ad09 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/96c5ae85acf24480b8e8dee445a3ad09 2024-11-09T10:18:32,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9ab93d6e22254c899664426a4f169010 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9ab93d6e22254c899664426a4f169010 2024-11-09T10:18:32,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/78fd1dd6f2314e468d8e8aef172ed50c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/78fd1dd6f2314e468d8e8aef172ed50c 2024-11-09T10:18:32,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/6cd2a2a6bfe849eeaa23d1f712030e67 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/6cd2a2a6bfe849eeaa23d1f712030e67 2024-11-09T10:18:32,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/347f7aca24324496a3aa921de6b4832c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/347f7aca24324496a3aa921de6b4832c 2024-11-09T10:18:32,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/dfde14536f15484398485bd2ec8296a6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/dfde14536f15484398485bd2ec8296a6 2024-11-09T10:18:32,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/b2c82d3b7c0a4bb68d6636ea2708e8c5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/b2c82d3b7c0a4bb68d6636ea2708e8c5 2024-11-09T10:18:32,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/5e82549f4cdc4521ab7c44130aa58358 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/5e82549f4cdc4521ab7c44130aa58358 2024-11-09T10:18:32,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a8e2d2be802b4658b19c5deef8021a91 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a8e2d2be802b4658b19c5deef8021a91 2024-11-09T10:18:32,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/af80eda4b8b94e648b39c4b19e514545 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/af80eda4b8b94e648b39c4b19e514545 2024-11-09T10:18:32,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/7b34129a600d4b48b3b632f1140fd7c7 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/7b34129a600d4b48b3b632f1140fd7c7 2024-11-09T10:18:32,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e06468637b0e47d0bdd27844262a7fec to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e06468637b0e47d0bdd27844262a7fec 2024-11-09T10:18:32,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/cab9045c37fe4db3b03466f19f25d5d3 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/cab9045c37fe4db3b03466f19f25d5d3 2024-11-09T10:18:32,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e4410e5d6bc04c39945af4defa557eca to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/e4410e5d6bc04c39945af4defa557eca 2024-11-09T10:18:32,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/0686517d9003419f98032a9685d2742d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/0686517d9003419f98032a9685d2742d 2024-11-09T10:18:32,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3657361e0b3f4e6a85c840f8385f8a1d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3657361e0b3f4e6a85c840f8385f8a1d 2024-11-09T10:18:32,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2030a1cab58545dca2bb240602e65d82 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2030a1cab58545dca2bb240602e65d82 2024-11-09T10:18:32,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/20f7b7012dfb4cd89237ab9d2aaa4e7c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/20f7b7012dfb4cd89237ab9d2aaa4e7c 2024-11-09T10:18:32,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/02cf8f2887e04f9ca8f4780651566c26 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/02cf8f2887e04f9ca8f4780651566c26 2024-11-09T10:18:32,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a7891640caec4f6db5438b16dbf6ada8 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/a7891640caec4f6db5438b16dbf6ada8 2024-11-09T10:18:32,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/561cc5e173c94cf79e45896cb22dd0bb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/561cc5e173c94cf79e45896cb22dd0bb 2024-11-09T10:18:32,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/acc67e6d15184a23b56c85e76e60de9f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/acc67e6d15184a23b56c85e76e60de9f 2024-11-09T10:18:32,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2fadc15ee00e4e2088e8964bf0fefd68 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2fadc15ee00e4e2088e8964bf0fefd68 2024-11-09T10:18:32,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2362dcbf6c504ca292a0b7bb97dc4980 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/2362dcbf6c504ca292a0b7bb97dc4980 2024-11-09T10:18:32,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/67bee43d8ce24fb3a671fcd1bc7a5006 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/67bee43d8ce24fb3a671fcd1bc7a5006 2024-11-09T10:18:32,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3bd5aca5e852490683107c70ce3eac8b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/3bd5aca5e852490683107c70ce3eac8b 2024-11-09T10:18:32,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/58e145d8dc8c4768b01b7f36f1962103 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/58e145d8dc8c4768b01b7f36f1962103 2024-11-09T10:18:32,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bdef70ec1511475197eeae351c080210, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/4e1674748b39499f8b46eb3b4580ed06, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/31eb3bb861464c39808d1b5f6c17ec25, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/b8d465a690554c4ba37bf637555f6399, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/46b7b781f22b45a4a85ba3cce5ac0c5c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c9991d57696f45e3a32ef44d6e680ba1, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/57f29e7b3de34a4c9a0305efe1319abd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8f778a343f88417480e0af14a0bffb04, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/71edd54fad29451c87c4f05e3b121537, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/e65ad687d0024fec97c97fb6c3777e5f, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/a431018662244275a39cda8c258b2f2d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bef15c122333425ab232adecd7c6ba94, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/83cbbe84d51d46d7ad1b4b422c2af194, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/defb378623754cdc8092439e5cb7f39e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/f542b2461dd1481f9e71df17fb53a98c, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ad63d5d47e80493da0a3895060a5c270, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c348ce33d8ad42f79063ca73cbed8f00, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/88f0252bca79468eb9a5c87867db99b6, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/583dfc190f524edfbc96bd9300630533, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e8349010438414dae8ef0ad037f2e2d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8596f37bdb704a29ae58a1328c61433e, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/15f566f34cc94553aa4fc089cc39cf5d, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/0a7f029adf064cfd9d0dc90d3f4a55bd, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/9a1dca3e57fb4135bf4e789d8731d914, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/fba0ddd8246f444ba2fa492a69a55a91, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ba0cb060bd1b494ea049da6145e993ad] to archive 2024-11-09T10:18:32,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T10:18:32,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bdef70ec1511475197eeae351c080210 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bdef70ec1511475197eeae351c080210 2024-11-09T10:18:32,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/4e1674748b39499f8b46eb3b4580ed06 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/4e1674748b39499f8b46eb3b4580ed06 2024-11-09T10:18:32,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/31eb3bb861464c39808d1b5f6c17ec25 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/31eb3bb861464c39808d1b5f6c17ec25 2024-11-09T10:18:32,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c4b21f2c0a3e4642a3eeeeccfaeeb2d9 2024-11-09T10:18:32,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/b8d465a690554c4ba37bf637555f6399 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/b8d465a690554c4ba37bf637555f6399 2024-11-09T10:18:32,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/46b7b781f22b45a4a85ba3cce5ac0c5c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/46b7b781f22b45a4a85ba3cce5ac0c5c 2024-11-09T10:18:32,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c9991d57696f45e3a32ef44d6e680ba1 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c9991d57696f45e3a32ef44d6e680ba1 2024-11-09T10:18:32,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/57f29e7b3de34a4c9a0305efe1319abd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/57f29e7b3de34a4c9a0305efe1319abd 2024-11-09T10:18:32,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8f778a343f88417480e0af14a0bffb04 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8f778a343f88417480e0af14a0bffb04 2024-11-09T10:18:32,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/71edd54fad29451c87c4f05e3b121537 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/71edd54fad29451c87c4f05e3b121537 2024-11-09T10:18:32,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/e65ad687d0024fec97c97fb6c3777e5f to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/e65ad687d0024fec97c97fb6c3777e5f 2024-11-09T10:18:32,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/a431018662244275a39cda8c258b2f2d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/a431018662244275a39cda8c258b2f2d 2024-11-09T10:18:32,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bef15c122333425ab232adecd7c6ba94 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/bef15c122333425ab232adecd7c6ba94 2024-11-09T10:18:32,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/83cbbe84d51d46d7ad1b4b422c2af194 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/83cbbe84d51d46d7ad1b4b422c2af194 2024-11-09T10:18:32,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/defb378623754cdc8092439e5cb7f39e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/defb378623754cdc8092439e5cb7f39e 2024-11-09T10:18:32,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/f542b2461dd1481f9e71df17fb53a98c to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/f542b2461dd1481f9e71df17fb53a98c 2024-11-09T10:18:32,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ad63d5d47e80493da0a3895060a5c270 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ad63d5d47e80493da0a3895060a5c270 2024-11-09T10:18:32,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c348ce33d8ad42f79063ca73cbed8f00 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/c348ce33d8ad42f79063ca73cbed8f00 2024-11-09T10:18:32,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/88f0252bca79468eb9a5c87867db99b6 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/88f0252bca79468eb9a5c87867db99b6 2024-11-09T10:18:32,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/583dfc190f524edfbc96bd9300630533 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/583dfc190f524edfbc96bd9300630533 2024-11-09T10:18:32,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e8349010438414dae8ef0ad037f2e2d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e8349010438414dae8ef0ad037f2e2d 2024-11-09T10:18:32,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8596f37bdb704a29ae58a1328c61433e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/8596f37bdb704a29ae58a1328c61433e 2024-11-09T10:18:32,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/15f566f34cc94553aa4fc089cc39cf5d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/15f566f34cc94553aa4fc089cc39cf5d 2024-11-09T10:18:32,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/0a7f029adf064cfd9d0dc90d3f4a55bd to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/0a7f029adf064cfd9d0dc90d3f4a55bd 2024-11-09T10:18:32,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/9a1dca3e57fb4135bf4e789d8731d914 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/9a1dca3e57fb4135bf4e789d8731d914 2024-11-09T10:18:32,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/fba0ddd8246f444ba2fa492a69a55a91 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/fba0ddd8246f444ba2fa492a69a55a91 2024-11-09T10:18:32,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ba0cb060bd1b494ea049da6145e993ad to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/ba0cb060bd1b494ea049da6145e993ad 2024-11-09T10:18:32,514 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/recovered.edits/686.seqid, newMaxSeqId=686, maxSeqId=4 2024-11-09T10:18:32,515 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. 2024-11-09T10:18:32,515 DEBUG [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for be37ddd6456db463b3a7fa388151b50e: Waiting for close lock at 1731147510799Running coprocessor pre-close hooks at 1731147510799Disabling compacts and flushes for region at 1731147510799Disabling writes for close at 1731147510800 (+1 ms)Obtaining lock to block concurrent updates at 1731147510800Preparing flush snapshotting stores in be37ddd6456db463b3a7fa388151b50e at 1731147510800Finished memstore snapshotting TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731147510800Flushing stores of TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e. at 1731147510800Flushing be37ddd6456db463b3a7fa388151b50e/A: creating writer at 1731147510801 (+1 ms)Flushing mob file be37ddd6456db463b3a7fa388151b50e/A: appending metadata at 1731147510804 (+3 ms)Flushing mob file be37ddd6456db463b3a7fa388151b50e/A: closing flushed file at 1731147510804Flushing be37ddd6456db463b3a7fa388151b50e/A: appending metadata at 1731147511215 (+411 ms)Flushing be37ddd6456db463b3a7fa388151b50e/A: closing flushed file at 1731147511216 (+1 ms)Flushing be37ddd6456db463b3a7fa388151b50e/B: creating writer at 1731147511622 (+406 ms)Flushing be37ddd6456db463b3a7fa388151b50e/B: appending metadata at 1731147511624 (+2 ms)Flushing be37ddd6456db463b3a7fa388151b50e/B: closing flushed file at 1731147511624Flushing be37ddd6456db463b3a7fa388151b50e/C: creating writer at 1731147512032 (+408 ms)Flushing be37ddd6456db463b3a7fa388151b50e/C: appending metadata at 1731147512034 (+2 ms)Flushing be37ddd6456db463b3a7fa388151b50e/C: closing flushed file at 1731147512034Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60dc97eb: reopening flushed file at 1731147512441 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35f4eb05: reopening flushed file at 1731147512444 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59b5f646: reopening flushed file at 1731147512447 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for be37ddd6456db463b3a7fa388151b50e in 1650ms, sequenceid=683, compaction requested=true at 1731147512450 (+3 ms)Writing region close event to WAL at 1731147512512 (+62 ms)Running coprocessor post-close hooks at 1731147512515 (+3 ms)Closed at 1731147512515 2024-11-09T10:18:32,516 INFO [RS_CLOSE_REGION-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=be37ddd6456db463b3a7fa388151b50e, regionState=CLOSED 2024-11-09T10:18:32,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 because future has completed 2024-11-09T10:18:32,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-09T10:18:32,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure be37ddd6456db463b3a7fa388151b50e, server=e067b2c2c7b3,38427,1731147328756 in 1.8700 sec 2024-11-09T10:18:32,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-09T10:18:32,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be37ddd6456db463b3a7fa388151b50e, UNASSIGN in 1.8730 sec 2024-11-09T10:18:32,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-09T10:18:32,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8760 sec 2024-11-09T10:18:32,523 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731147512522"}]},"ts":"1731147512522"} 2024-11-09T10:18:32,524 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T10:18:32,524 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T10:18:32,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-09T10:18:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T10:18:32,789 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:32,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T10:18:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:32,790 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T10:18:32,791 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:32,793 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,794 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C, FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/recovered.edits] 2024-11-09T10:18:32,796 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/889856d3842f46cca33cbfe07d63ff98 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/889856d3842f46cca33cbfe07d63ff98 2024-11-09T10:18:32,797 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8e7ef11bbbf64de681719fb8a3147ff5 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/8e7ef11bbbf64de681719fb8a3147ff5 2024-11-09T10:18:32,798 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/e85dbdc846094a10bba78b0f01226b3e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/A/e85dbdc846094a10bba78b0f01226b3e 2024-11-09T10:18:32,800 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/4ae8724bfe4b4b579a94594d9f9f304d to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/4ae8724bfe4b4b579a94594d9f9f304d 2024-11-09T10:18:32,801 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9c0f6c95614d4e528123dce74cdccbdb to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/9c0f6c95614d4e528123dce74cdccbdb 2024-11-09T10:18:32,802 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/d4fd21f66b444b8182f21de17287f9f2 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/B/d4fd21f66b444b8182f21de17287f9f2 2024-11-09T10:18:32,803 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/3e3631205ef74503a5e5a67c12eebd56 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/3e3631205ef74503a5e5a67c12eebd56 2024-11-09T10:18:32,804 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e50da3947614ee88aed783fd56a9b0b to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/5e50da3947614ee88aed783fd56a9b0b 2024-11-09T10:18:32,805 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/6817071ef7794cb3b19444bc35e30c02 to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/C/6817071ef7794cb3b19444bc35e30c02 2024-11-09T10:18:32,807 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/recovered.edits/686.seqid to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e/recovered.edits/686.seqid 2024-11-09T10:18:32,807 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/default/TestAcidGuarantees/be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,807 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T10:18:32,808 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T10:18:32,808 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T10:18:32,810 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109024100c1e07a4471b914f54e13398e0d_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109024100c1e07a4471b914f54e13398e0d_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,811 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109097e498f11354d9cae26e4daaa2ce3e8_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109097e498f11354d9cae26e4daaa2ce3e8_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,812 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d2dd079100d4506800fcc1a3a9422e0_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d2dd079100d4506800fcc1a3a9422e0_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,812 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090f9b318f543648d4a59c123bc96d20c0_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090f9b318f543648d4a59c123bc96d20c0_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,813 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092395030e43ed4649b27c695027baae07_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092395030e43ed4649b27c695027baae07_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,814 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109260b63e6e86d4fad871778e56a406bde_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109260b63e6e86d4fad871778e56a406bde_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,814 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110943cd2ee14b254323be6446c5b47683cb_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110943cd2ee14b254323be6446c5b47683cb_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,815 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094cf015777ed74742a77a09077170e3ec_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094cf015777ed74742a77a09077170e3ec_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,816 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110953aecfa6aa7e490e9068873b366e158c_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110953aecfa6aa7e490e9068873b366e158c_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,816 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110955760977765a49bbb3c51a7def156c53_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110955760977765a49bbb3c51a7def156c53_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,817 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095e7d85732e934a0aa2c52a532f1f8326_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095e7d85732e934a0aa2c52a532f1f8326_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,818 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096f807253bf304228a66179f9832f967d_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096f807253bf304228a66179f9832f967d_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,818 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b64e148d5914eb8943d62c63cd55e38_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b64e148d5914eb8943d62c63cd55e38_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,819 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110983b0dfaa59094ca0bca812f1c436037d_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110983b0dfaa59094ca0bca812f1c436037d_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,820 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098794bb230675493b8e82052204c7b317_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098794bb230675493b8e82052204c7b317_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,821 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099c01420b04a64a1e89d445b42261fd66_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099c01420b04a64a1e89d445b42261fd66_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,821 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b5e7e579ae29424883615f4e6a529c7b_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b5e7e579ae29424883615f4e6a529c7b_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,822 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109da2817c451594814bba8e928e2cc46a6_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109da2817c451594814bba8e928e2cc46a6_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,823 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e9880089672745219d35b7c45173255e_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e9880089672745219d35b7c45173255e_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,823 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f298b43646e645db858d38c0040b37b1_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f298b43646e645db858d38c0040b37b1_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,824 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f31a071359214b63b6349e96113fb7f5_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f31a071359214b63b6349e96113fb7f5_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,825 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fbe3e7723cd1470393388bfbecb3ec30_be37ddd6456db463b3a7fa388151b50e to hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fbe3e7723cd1470393388bfbecb3ec30_be37ddd6456db463b3a7fa388151b50e 2024-11-09T10:18:32,825 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T10:18:32,827 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:32,828 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T10:18:32,830 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T10:18:32,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:32,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T10:18:32,831 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731147512831"}]},"ts":"9223372036854775807"} 2024-11-09T10:18:32,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T10:18:32,832 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => be37ddd6456db463b3a7fa388151b50e, NAME => 'TestAcidGuarantees,,1731147480615.be37ddd6456db463b3a7fa388151b50e.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T10:18:32,832 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T10:18:32,832 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731147512832"}]},"ts":"9223372036854775807"} 2024-11-09T10:18:32,833 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T10:18:32,834 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T10:18:32,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-09T10:18:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T10:18:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T10:18:32,899 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T10:18:32,906 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=549 (was 521) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=943 (was 1347) 2024-11-09T10:18:32,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T10:18:32,906 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:18:32,906 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:18:32,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:32,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:32,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T10:18:32,907 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T10:18:32,907 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-09T10:18:32,907 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e067b2c2c7b3,42133,1731147327850 2024-11-09T10:18:32,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T10:18:32,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T10:18:32,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:18:32,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:18:32,968 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T10:18:32,969 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T10:18:32,969 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T10:18:32,969 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:18:32,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:32,969 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T10:18:32,969 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e067b2c2c7b3,38427,1731147328756' ***** 2024-11-09T10:18:32,969 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T10:18:32,970 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(959): stopping server e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e067b2c2c7b3:38427. 2024-11-09T10:18:32,970 DEBUG [RS:0;e067b2c2c7b3:38427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T10:18:32,970 DEBUG [RS:0;e067b2c2c7b3:38427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T10:18:32,970 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T10:18:32,971 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-09T10:18:32,971 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-09T10:18:32,971 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T10:18:32,971 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T10:18:32,971 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T10:18:32,971 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T10:18:32,971 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T10:18:32,971 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T10:18:32,971 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-09T10:18:32,992 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/info/3585b87b2d9242da9b568010957bb084 is 95, key is TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658./info:/1731147359076/DeleteFamily/seqid=0 2024-11-09T10:18:32,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742456_1632 (size=6514) 2024-11-09T10:18:33,001 INFO [regionserver/e067b2c2c7b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T10:18:33,171 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T10:18:33,186 INFO [regionserver/e067b2c2c7b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T10:18:33,186 INFO [regionserver/e067b2c2c7b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T10:18:33,372 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T10:18:33,396 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/info/3585b87b2d9242da9b568010957bb084 2024-11-09T10:18:33,418 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/ns/b5bb102edbdc443a90010bb4916d7909 is 93, key is TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658./ns:/1731147359032/DeleteFamily/seqid=0 2024-11-09T10:18:33,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742457_1633 (size=5906) 2024-11-09T10:18:33,421 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/ns/b5bb102edbdc443a90010bb4916d7909 2024-11-09T10:18:33,438 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/rep_barrier/f937f43150ac4d789d733fa6e98fb5a6 is 102, key is TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658./rep_barrier:/1731147359032/DeleteFamily/seqid=0 2024-11-09T10:18:33,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742458_1634 (size=6025) 2024-11-09T10:18:33,572 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T10:18:33,772 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T10:18:33,841 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/rep_barrier/f937f43150ac4d789d733fa6e98fb5a6 2024-11-09T10:18:33,859 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/table/eac7b2562aed4ee6ac1445e952de323a is 96, key is TestAcidGuarantees,,1731147333094.cac2483be99c556ef585845f10246658./table:/1731147359032/DeleteFamily/seqid=0 2024-11-09T10:18:33,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742459_1635 (size=5936) 2024-11-09T10:18:33,972 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T10:18:34,172 DEBUG [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T10:18:34,265 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/table/eac7b2562aed4ee6ac1445e952de323a 2024-11-09T10:18:34,268 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/info/3585b87b2d9242da9b568010957bb084 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/info/3585b87b2d9242da9b568010957bb084 2024-11-09T10:18:34,271 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/info/3585b87b2d9242da9b568010957bb084, entries=12, sequenceid=90, filesize=6.4 K 2024-11-09T10:18:34,271 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/ns/b5bb102edbdc443a90010bb4916d7909 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/ns/b5bb102edbdc443a90010bb4916d7909 2024-11-09T10:18:34,274 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/ns/b5bb102edbdc443a90010bb4916d7909, entries=8, sequenceid=90, filesize=5.8 K 2024-11-09T10:18:34,274 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/rep_barrier/f937f43150ac4d789d733fa6e98fb5a6 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/rep_barrier/f937f43150ac4d789d733fa6e98fb5a6 2024-11-09T10:18:34,277 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/rep_barrier/f937f43150ac4d789d733fa6e98fb5a6, entries=6, sequenceid=90, filesize=5.9 K 2024-11-09T10:18:34,277 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/.tmp/table/eac7b2562aed4ee6ac1445e952de323a as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/table/eac7b2562aed4ee6ac1445e952de323a 2024-11-09T10:18:34,280 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/table/eac7b2562aed4ee6ac1445e952de323a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-09T10:18:34,281 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1310ms, sequenceid=90, compaction requested=false 2024-11-09T10:18:34,287 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-09T10:18:34,287 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T10:18:34,287 INFO [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T10:18:34,287 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731147512971Running coprocessor pre-close hooks at 1731147512971Disabling compacts and flushes for region at 1731147512971Disabling writes for close at 1731147512971Obtaining lock to block concurrent updates at 1731147512971Preparing flush snapshotting stores in 1588230740 at 1731147512971Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731147512972 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731147512972Flushing 1588230740/info: creating writer at 1731147512973 (+1 ms)Flushing 1588230740/info: appending metadata at 1731147512992 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731147512992Flushing 1588230740/ns: creating writer at 1731147513400 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731147513417 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731147513418 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1731147513424 (+6 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731147513438 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731147513438Flushing 1588230740/table: creating writer at 1731147513845 (+407 ms)Flushing 1588230740/table: appending metadata at 1731147513859 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731147513859Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2afd7007: reopening flushed file at 1731147514268 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4807fadd: reopening flushed file at 1731147514271 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c4fb018: reopening flushed file at 1731147514274 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@245b9bfa: reopening flushed file at 1731147514277 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1310ms, sequenceid=90, compaction requested=false at 1731147514281 (+4 ms)Writing region close event to WAL at 1731147514284 (+3 ms)Running coprocessor post-close hooks at 1731147514287 (+3 ms)Closed at 1731147514287 2024-11-09T10:18:34,287 DEBUG [RS_CLOSE_META-regionserver/e067b2c2c7b3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T10:18:34,373 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(976): stopping server e067b2c2c7b3,38427,1731147328756; all regions closed. 2024-11-09T10:18:34,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741834_1010 (size=25344) 2024-11-09T10:18:34,379 DEBUG [RS:0;e067b2c2c7b3:38427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/oldWALs 2024-11-09T10:18:34,379 INFO [RS:0;e067b2c2c7b3:38427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e067b2c2c7b3%2C38427%2C1731147328756.meta:.meta(num 1731147332496) 2024-11-09T10:18:34,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741832_1008 (size=17636871) 2024-11-09T10:18:34,383 DEBUG [RS:0;e067b2c2c7b3:38427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/oldWALs 2024-11-09T10:18:34,383 INFO [RS:0;e067b2c2c7b3:38427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e067b2c2c7b3%2C38427%2C1731147328756:(num 1731147331301) 2024-11-09T10:18:34,383 DEBUG [RS:0;e067b2c2c7b3:38427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T10:18:34,383 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T10:18:34,383 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T10:18:34,383 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.ChoreService(370): Chore service for: regionserver/e067b2c2c7b3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T10:18:34,384 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T10:18:34,384 INFO [regionserver/e067b2c2c7b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T10:18:34,385 INFO [RS:0;e067b2c2c7b3:38427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38427 2024-11-09T10:18:34,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e067b2c2c7b3,38427,1731147328756 2024-11-09T10:18:34,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T10:18:34,452 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T10:18:34,463 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e067b2c2c7b3,38427,1731147328756] 2024-11-09T10:18:34,473 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e067b2c2c7b3,38427,1731147328756 already deleted, retry=false 2024-11-09T10:18:34,473 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e067b2c2c7b3,38427,1731147328756 expired; onlineServers=0 2024-11-09T10:18:34,473 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e067b2c2c7b3,42133,1731147327850' ***** 2024-11-09T10:18:34,473 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T10:18:34,474 INFO [M:0;e067b2c2c7b3:42133 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T10:18:34,474 INFO [M:0;e067b2c2c7b3:42133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T10:18:34,474 DEBUG [M:0;e067b2c2c7b3:42133 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T10:18:34,474 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T10:18:34,474 DEBUG [M:0;e067b2c2c7b3:42133 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T10:18:34,474 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster-HFileCleaner.large.0-1731147330999 {}] cleaner.HFileCleaner(306): Exit Thread[master/e067b2c2c7b3:0:becomeActiveMaster-HFileCleaner.large.0-1731147330999,5,FailOnTimeoutGroup] 2024-11-09T10:18:34,474 DEBUG [master/e067b2c2c7b3:0:becomeActiveMaster-HFileCleaner.small.0-1731147331000 {}] cleaner.HFileCleaner(306): Exit Thread[master/e067b2c2c7b3:0:becomeActiveMaster-HFileCleaner.small.0-1731147331000,5,FailOnTimeoutGroup] 2024-11-09T10:18:34,474 INFO [M:0;e067b2c2c7b3:42133 {}] hbase.ChoreService(370): Chore service for: master/e067b2c2c7b3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T10:18:34,474 INFO [M:0;e067b2c2c7b3:42133 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T10:18:34,474 DEBUG [M:0;e067b2c2c7b3:42133 {}] master.HMaster(1795): Stopping service threads 2024-11-09T10:18:34,474 INFO [M:0;e067b2c2c7b3:42133 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T10:18:34,474 INFO [M:0;e067b2c2c7b3:42133 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T10:18:34,474 ERROR [M:0;e067b2c2c7b3:42133 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-09T10:18:34,475 INFO [M:0;e067b2c2c7b3:42133 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T10:18:34,475 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T10:18:34,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T10:18:34,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T10:18:34,484 DEBUG [M:0;e067b2c2c7b3:42133 {}] zookeeper.ZKUtil(347): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T10:18:34,484 WARN [M:0;e067b2c2c7b3:42133 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T10:18:34,485 INFO [M:0;e067b2c2c7b3:42133 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/.lastflushedseqids 2024-11-09T10:18:34,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742460_1636 (size=72) 2024-11-09T10:18:34,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T10:18:34,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38427-0x1011f17e1cc0001, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T10:18:34,563 INFO [RS:0;e067b2c2c7b3:38427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T10:18:34,564 INFO [RS:0;e067b2c2c7b3:38427 {}] regionserver.HRegionServer(1031): Exiting; stopping=e067b2c2c7b3,38427,1731147328756; zookeeper connection closed. 2024-11-09T10:18:34,564 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@573d316b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@573d316b 2024-11-09T10:18:34,564 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T10:18:34,893 INFO [M:0;e067b2c2c7b3:42133 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T10:18:34,893 INFO [M:0;e067b2c2c7b3:42133 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T10:18:34,893 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T10:18:34,893 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:18:34,893 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:18:34,894 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T10:18:34,894 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:18:34,894 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=790.23 KB heapSize=969.40 KB 2024-11-09T10:18:34,907 DEBUG [M:0;e067b2c2c7b3:42133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/973ea4c3d43f43dd81c43b2e45b506eb is 82, key is hbase:meta,,1/info:regioninfo/1731147332575/Put/seqid=0 2024-11-09T10:18:34,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742461_1637 (size=5672) 2024-11-09T10:18:35,310 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2218 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/973ea4c3d43f43dd81c43b2e45b506eb 2024-11-09T10:18:35,334 DEBUG [M:0;e067b2c2c7b3:42133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb551cb41e414b47983ab1802693c690 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731147483700/Put/seqid=0 2024-11-09T10:18:35,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742462_1638 (size=45423) 2024-11-09T10:18:35,363 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=789.67 KB at sequenceid=2218 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb551cb41e414b47983ab1802693c690 2024-11-09T10:18:35,369 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cb551cb41e414b47983ab1802693c690 2024-11-09T10:18:35,394 DEBUG [M:0;e067b2c2c7b3:42133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbe8d0e48a3e4077a4ac48d4a85c8e82 is 69, key is e067b2c2c7b3,38427,1731147328756/rs:state/1731147331070/Put/seqid=0 2024-11-09T10:18:35,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073742463_1639 (size=5156) 2024-11-09T10:18:35,404 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2218 (bloomFilter=true), to=hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbe8d0e48a3e4077a4ac48d4a85c8e82 2024-11-09T10:18:35,409 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/973ea4c3d43f43dd81c43b2e45b506eb as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/973ea4c3d43f43dd81c43b2e45b506eb 2024-11-09T10:18:35,413 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/973ea4c3d43f43dd81c43b2e45b506eb, entries=8, sequenceid=2218, filesize=5.5 K 2024-11-09T10:18:35,414 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb551cb41e414b47983ab1802693c690 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cb551cb41e414b47983ab1802693c690 2024-11-09T10:18:35,417 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cb551cb41e414b47983ab1802693c690 2024-11-09T10:18:35,417 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cb551cb41e414b47983ab1802693c690, entries=170, sequenceid=2218, filesize=44.4 K 2024-11-09T10:18:35,418 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbe8d0e48a3e4077a4ac48d4a85c8e82 as hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cbe8d0e48a3e4077a4ac48d4a85c8e82 2024-11-09T10:18:35,422 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46023/user/jenkins/test-data/8fb9e705-a704-f137-53fb-6dc40c2bff51/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cbe8d0e48a3e4077a4ac48d4a85c8e82, entries=1, sequenceid=2218, filesize=5.0 K 2024-11-09T10:18:35,423 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(3140): Finished flush of dataSize ~790.23 KB/809196, heapSize ~969.10 KB/992360, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 529ms, sequenceid=2218, compaction requested=false 2024-11-09T10:18:35,428 INFO [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T10:18:35,428 DEBUG [M:0;e067b2c2c7b3:42133 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731147514893Disabling compacts and flushes for region at 1731147514893Disabling writes for close at 1731147514894 (+1 ms)Obtaining lock to block concurrent updates at 1731147514894Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731147514894Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=809196, getHeapSize=992600, getOffHeapSize=0, getCellsCount=2413 at 1731147514894Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731147514894Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731147514894Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731147514906 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731147514906Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731147515314 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731147515333 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731147515333Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731147515369 (+36 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731147515393 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731147515393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67f2e8fc: reopening flushed file at 1731147515408 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@582a5330: reopening flushed file at 1731147515413 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44c08d4f: reopening flushed file at 1731147515418 (+5 ms)Finished flush of dataSize ~790.23 KB/809196, heapSize ~969.10 KB/992360, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 529ms, sequenceid=2218, compaction requested=false at 1731147515423 (+5 ms)Writing region close event to WAL at 1731147515428 (+5 ms)Closed at 1731147515428 2024-11-09T10:18:35,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37425 is added to blk_1073741830_1006 (size=954565) 2024-11-09T10:18:35,437 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T10:18:35,438 INFO [M:0;e067b2c2c7b3:42133 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T10:18:35,438 INFO [M:0;e067b2c2c7b3:42133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42133 2024-11-09T10:18:35,438 INFO [M:0;e067b2c2c7b3:42133 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T10:18:35,605 INFO [M:0;e067b2c2c7b3:42133 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T10:18:35,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T10:18:35,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42133-0x1011f17e1cc0000, quorum=127.0.0.1:59425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T10:18:35,609 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T10:18:35,611 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T10:18:35,612 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T10:18:35,612 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T10:18:35,612 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/hadoop.log.dir/,STOPPED} 2024-11-09T10:18:35,614 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T10:18:35,614 WARN [BP-2048340984-172.17.0.3-1731147323772 heartbeating to localhost/127.0.0.1:46023 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T10:18:35,614 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T10:18:35,614 WARN [BP-2048340984-172.17.0.3-1731147323772 heartbeating to localhost/127.0.0.1:46023 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2048340984-172.17.0.3-1731147323772 (Datanode Uuid a9670b94-06a3-4eea-93ae-1a8a605d573e) service to localhost/127.0.0.1:46023 2024-11-09T10:18:35,618 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4/data/data1/current/BP-2048340984-172.17.0.3-1731147323772 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T10:18:35,618 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/cluster_5815f560-f06b-7c06-6d3a-fc86b0305bf4/data/data2/current/BP-2048340984-172.17.0.3-1731147323772 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T10:18:35,619 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T10:18:35,659 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T10:18:35,660 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T10:18:35,660 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T10:18:35,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T10:18:35,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ac9cf06d-1c7f-5395-c005-d6c4cf2e63e2/hadoop.log.dir/,STOPPED} 2024-11-09T10:18:35,674 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T10:18:35,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down